
#include "mmio.h"
#include "reg_soc.h"
#include "ddr_sys.h"
#include "ddr_init.h"
#include "bitwise_ops.h"
#include "cvx16_dram_cap_check.h"
#include "cvx16_pinmux.h"
#include "ddr_pkg_info.h"
#include "regconfig.h"



extern void udelay( uint32_t usec );
#define opdelay(_x) udelay((_x)/1000)

#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wunused-but-set-variable"


uint32_t rddata;

/*
 * unused
void check_rd32(uintptr_t addr, uint32_t expected)
{
	// uint32_t rdata;
	// rdata = mmio_rd32(addr);
	// if (rdata != expected)
	//	mmio_wr32(TERM_SIM_ADDR, 0x0B0B0B0B); // fail
}
*/

void ddr_debug_num_write(void)
{
	// debug_seqnum = debug_seqnum+1 ;
	// mmio_wr32(4*(186 + PHY_BASE_ADDR)+CADENCE_PHYD,(debug_seqnum<<8));
	// if (debug_seqnum ==255){ ;
	// debug_seqnum1 = debug_seqnum1+1 ;
	// mmio_wr32(4*(186 + PHY_BASE_ADDR)+CADENCE_PHYD,(debug_seqnum1<<8));
	// debug_seqnum = 0 ;
	// } ;
}

void load_ddr_patch_set_data(void)
{
	
#ifndef DDR2_3
	for (int i = 0; i < ddr_patch_regs_count; i++) {
		uint32_t addr = ddr_patch_regs[i].addr;
		uint32_t mask = ddr_patch_regs[i].mask;
		uint32_t val = ddr_patch_regs[i].val;
		uint32_t orig;

		orig = mmio_rd32(addr);
		orig &= ~mask;
		mmio_wr32(addr, orig | (val & mask));
	}
#else
	if (get_ddr_type() == DDR_TYPE_DDR3) {
		for (int i = 0; i < ddr3_1866_patch_regs_count; i++) {
			uint32_t addr = ddr3_1866_patch_regs[i].addr;
			uint32_t mask = ddr3_1866_patch_regs[i].mask;
			uint32_t val = ddr3_1866_patch_regs[i].val;

			uint32_t orig;

			orig = mmio_rd32(addr);
			orig &= ~mask;
			mmio_wr32(addr, orig | (val & mask));
		}
	} else if (get_ddr_type() == DDR_TYPE_DDR2) {
		for (int i = 0; i < ddr2_1333_patch_regs_count; i++) {
			uint32_t addr = ddr2_1333_patch_regs[i].addr;
			uint32_t mask = ddr2_1333_patch_regs[i].mask;
			uint32_t val = ddr2_1333_patch_regs[i].val;

			uint32_t orig;

			orig = mmio_rd32(addr);
			orig &= ~mask;
			mmio_wr32(addr, orig | (val & mask));
		}
	} else {
		NOTICE("error ddr type.\n");
	}
#endif
}

void ddr_patch_set(void)
{
	load_ddr_patch_set_data();

#ifdef DDR3_DBG
	mmio_wr32(0x005C + PHYD_BASE_ADDR, 0x00FE0000);//wrlvl response only DQ0
#endif//DDR3_DBG
}

void cvx16_rdvld_train(void)
{
	uint32_t byte0_vld;
	uint32_t byte1_vld;
	uint32_t rdvld_offset;
	uint32_t bist_result;
	uint64_t err_data_odd;
	uint64_t err_data_even;

	cvx16_bist_wr_prbs_init();
	// cvx16_bist_wr_sram_init();

	byte0_vld = mmio_rd32(0x0B14 + PHYD_BASE_ADDR);
	byte1_vld = mmio_rd32(0x0B44 + PHYD_BASE_ADDR);
	rdvld_offset = mmio_rd32(0x0094 + PHYD_BASE_ADDR);
	rdvld_offset = get_bits_from_value(rdvld_offset, 3, 0);

	for (int i = 9; i > 1; i--) {
		byte0_vld = modified_bits_by_value(byte0_vld, i, 20, 16);
		mmio_wr32(0x0B14 + PHYD_BASE_ADDR, byte0_vld);
		byte1_vld = modified_bits_by_value(byte1_vld, i, 20, 16);
		mmio_wr32(0x0B44 + PHYD_BASE_ADDR, byte0_vld);
		cvx16_bist_start_check(&bist_result, &err_data_odd, &err_data_even);

		// KC_MSG(", bist_result = %x, err_data_odd = %x, err_data_even = %x\n",
		//         bist_result,err_data_odd,err_data_even);
		if (bist_result == 0) {
			KC_MSG("vld end = %x, sel = %x", i, i + 1 + rdvld_offset);
			i = i + 1 + rdvld_offset;
			byte0_vld = modified_bits_by_value(byte0_vld, i, 20, 16);
			mmio_wr32(0x0B14 + PHYD_BASE_ADDR, byte0_vld);
			byte1_vld = modified_bits_by_value(byte1_vld, i, 20, 16);
			mmio_wr32(0x0B44 + PHYD_BASE_ADDR, byte0_vld);
			break;
		}
	}
}


void cvx16_bist_wr_prbs_init(void)
{
	// reg [31:0] cmd[5:0];
	// reg [31:0] sram_st;
	// reg [31:0] sram_sp;
	uint32_t cmd[6];
	uint32_t sram_st;
	uint32_t sram_sp;
	int i;

	uartlog("%s\n", __func__);
	ddr_debug_wr32(0x23);
	ddr_debug_num_write();
	KC_MSG("bist_wr_prbs_init\n");

	// bist clock enable
	mmio_wr32(DDR_BIST_BASE + 0x0, 0x00060006);
	sram_st = 0;
	sram_sp = 511;
	// cmd queue
	//        op_code         start              stop        pattern    dq_inv     dm_inv    dq_rotate
	//        repeat
	cmd[0] = (1 << 30) | (sram_st << 21) | (sram_sp << 12) | (5 << 9) | (0 << 8) | (0 << 7) | (0 << 4) |
		 (0 << 0); // W  1~17  prbs  repeat0
	cmd[1] = (2 << 30) | (sram_st << 21) | (sram_sp << 12) | (5 << 9) | (0 << 8) | (0 << 7) | (0 << 4) |
		 (0 << 0); // R  1~17  prbs  repeat0
	cmd[2] = 0; // NOP
	cmd[3] = 0; // NOP
	cmd[4] = 0; // NOP
	cmd[5] = 0; // NOP
	// write cmd queue
	for (i = 0; i < 6; i = i + 1) {
		mmio_wr32(DDR_BIST_BASE + 0x40 + i * 4, cmd[i]);
	};
	// specified DDR space
	mmio_wr32(DDR_BIST_BASE + 0x10, 0x00000000);
	mmio_wr32(DDR_BIST_BASE + 0x14, 0x000fffff);
#ifdef X16_MODE
	// specified AXI address step
	mmio_wr32(DDR_BIST_BASE + 0x18, 0x00000004);
#else
	// specified AXI address step
	mmio_wr32(DDR_BIST_BASE + 0x18, 0x00000008);
#endif
	uartlog("bist_wr_prbs_init done\n");
	KC_MSG("bist_wr_prbs_init done\n");
}

void cvx16_bist_wr_sram_init(void)
{
	const int byte_per_page = 256;
	int axi_per_page = byte_per_page / 64;
	uint32_t cmd[6];
	uint32_t sram_st;
	uint32_t sram_sp;
	uint32_t fmax;
	uint32_t fmin;
	int i;

	uartlog("%s\n", __func__);
	ddr_debug_wr32(0x23);
	ddr_debug_num_write();
	KC_MSG("bist_wr_sram_init\n");

	// bist clock enable, axi_len 8
	mmio_wr32(DDR_BIST_BASE + 0x0, 0x000C000C);
	sram_st = 0;
	fmax = 15;
	fmin = 5;
	sram_st = 0;
	sram_sp = 9 * (fmin + fmax) * (fmax - fmin + 1) / 2 / 4 + (fmax - fmin + 1); // 8*f/4 -1
	KC_MSG("sram_sp = %x\n", sram_sp);

	// bist sso_period
	mmio_wr32(DDR_BIST_BASE + 0x24, (fmax << 8) + fmin);
	// cmd queue
	//        op_code         start              stop        pattern    dq_inv     dm_inv    dq_rotate
	//        repeat
	cmd[0] = (1 << 30) | (sram_st << 21) | (sram_sp << 12) | (6 << 9) | (0 << 8) | (0 << 7) | (0 << 4) |
		 (15 << 0); // W  1~17  sram  repeat0
	cmd[1] = (2 << 30) | (sram_st << 21) | (sram_sp << 12) | (6 << 9) | (0 << 8) | (0 << 7) | (0 << 4) |
		 (15 << 0); // R  1~17  sram  repeat0
	//       NOP idle
	cmd[2] = 0; // NOP
	//       GOTO      addr_not_reset loop_cnt
	cmd[3] = (3 << 30) | (0 << 20) | (1 << 0); // GOTO
	cmd[4] = 0; // NOP
	cmd[5] = 0; // NOP
	// write cmd queue
	for (i = 0; i < 6; i = i + 1) {
		mmio_wr32(DDR_BIST_BASE + 0x40 + i * 4, cmd[i]);
	};
	// specified DDR space
	mmio_wr32(DDR_BIST_BASE + 0x10, 0x00000000);
	mmio_wr32(DDR_BIST_BASE + 0x14, 0x000fffff);
#ifdef X16_MODE
	// specified AXI address step to 2KB
	mmio_wr32(DDR_BIST_BASE + 0x18, 2048 / axi_per_page / 16);
#else
	// TBD
#endif
	uartlog("bist_wr_sram_init done\n");
	KC_MSG("bist_wr_sram_init done\n");
}


void cvx16_bist_rdglvl_init(void)
{
	uint32_t cmd[6];
	uint32_t sram_st;
	uint32_t sram_sp;
	int i;

	uartlog("%s\n", __func__);
	ddr_debug_wr32(0x231);
	ddr_debug_num_write();
	KC_MSG("%s\n", __func__);

	// bist clock enable
	mmio_wr32(DDR_BIST_BASE + 0x0, 0x00060006);
	sram_st = 0;
	// sram_sp = 511;
	sram_sp = 3;
	// cmd queue
	//        op_code         start              stop        pattern    dq_inv     dm_inv    dq_rotate
	//        repeat
	// cmd[0] = (1 << 30) | (sram_st << 21) | (sram_sp << 12) | (5 << 9) | (0 << 8) | (0 << 7) | (0 << 4) |
	//          (0 << 0);  // W  1~17  prbs  repeat0
	cmd[0] = (2 << 30) | (sram_st << 21) | (sram_sp << 12) | (5 << 9) | (0 << 8) | (0 << 7) | (0 << 4) |
		 (0 << 0); // R  1~17  prbs  repeat0
	// cmd[1] = (2 << 30) | (sram_st << 21) | (sram_sp << 12) | (5 << 9) | (0 << 8) | (0 << 7) | (0 << 4) |
	//          (0 << 0);  // R  1~17  prbs  repeat0
	cmd[1] = 0; // NOP
	cmd[2] = 0; // NOP
	cmd[3] = 0; // NOP
	cmd[4] = 0; // NOP
	cmd[5] = 0; // NOP
	// write cmd queue
	for (i = 0; i < 6; i = i + 1) {
		mmio_wr32(DDR_BIST_BASE + 0x40 + i * 4, cmd[i]);
	};
	// specified DDR space
	mmio_wr32(DDR_BIST_BASE + 0x10, 0x00000000);
	mmio_wr32(DDR_BIST_BASE + 0x14, 0x000fffff);
#ifdef X16_MODE
	// specified AXI address step
	mmio_wr32(DDR_BIST_BASE + 0x18, 0x00000004);
#else
	// specified AXI address step
	mmio_wr32(DDR_BIST_BASE + 0x18, 0x00000008);
#endif
	uartlog("%s done\n", __func__);
	KC_MSG("%s done\n", __func__);
}

void cvx16_bist_rdlvl_init(uint32_t mode)
{
	uint32_t cmd[6];
	uint32_t sram_st;
	uint32_t sram_sp;
	uint32_t fmax;
	uint32_t fmin;
	// uint32_t sram_dq[1024];
	int i;

	// mode = 0x0  : MPR mode, DDR3 only.
	// mode = 0x1  : sram write/read continuous goto
	// mode = 0x2  : multi- bist write/read
	// mode = 0x12 : with Error enject,  multi- bist write/read
	uartlog("%s\n", __func__);
	ddr_debug_wr32(0x232);
	ddr_debug_num_write();
	KC_MSG("%s mode = %x\n", __func__, mode);

	// bist clock enable
	mmio_wr32(DDR_BIST_BASE + 0x0, 0x00060006);
	if (mode == 0x0) { // MPR mode
		sram_st = 0;
		// sram_sp = 511;
		sram_sp = 3;
		// cmd queue
		//        op_code         start              stop        pattern    dq_inv     dm_inv    dq_rotate
		//        repeat
		// cmd[0] = (1 << 30) | (sram_st << 21) | (sram_sp << 12) | (5 << 9) | (0 << 8) | (0 << 7) | (0 << 4) |
		//          (0 << 0);  // W  1~17  prbs  repeat0
		cmd[0] = (2 << 30) | (sram_st << 21) | (sram_sp << 12) | (5 << 9) | (0 << 8) | (0 << 7) | (0 << 4) |
			 (0 << 0); // R  1~17  prbs  repeat0
		// cmd[1] = (2 << 30) | (sram_st << 21) | (sram_sp << 12) | (5 << 9) | (0 << 8) | (0 << 7) | (0 << 4) |
		//          (0 << 0);  // R  1~17  prbs  repeat0
		cmd[1] = 0; // NOP
		cmd[2] = 0; // NOP
		cmd[3] = 0; // NOP
		cmd[4] = 0; // NOP
		cmd[5] = 0; // NOP

	} else if (mode == 0x1) { // sram write/read continuous goto

		fmax = 15;
		fmin = 5;
		sram_st = 0;
		sram_sp = 9 * (fmin + fmax) * (fmax - fmin + 1) / 2 / 4 + (fmax - fmin + 1); // 8*f/4 -1
		KC_MSG("sram_sp = %x\n", sram_sp);

		// bist sso_period
		mmio_wr32(DDR_BIST_BASE + 0x24, (fmax << 8) + fmin);

		// cmd queue
		//        op_code         start              stop        pattern    dq_inv     dm_inv    dq_rotate
		//        repeat
		cmd[0] = (1 << 30) | (sram_st << 21) | (511 << 12) | (5 << 9) | (0 << 8) | (0 << 7) | (0 << 4) |
			 (2 << 0); // W  1~17  prbs  repeat0
		cmd[1] = (2 << 30) | (sram_st << 21) | (511 << 12) | (5 << 9) | (0 << 8) | (0 << 7) | (0 << 4) |
			 (2 << 0); // R  1~17  prbs  repeat0
		cmd[2] = (1 << 30) | (sram_st << 21) | (sram_sp << 12) | (6 << 9) | (0 << 8) | (0 << 7) | (0 << 4) |
			 (2 << 0); // W  1~17  sram  repeat0
		cmd[3] = (2 << 30) | (sram_st << 21) | (sram_sp << 12) | (6 << 9) | (0 << 8) | (0 << 7) | (0 << 4) |
			 (2 << 0); // R  1~17  sram  repeat0
		//       GOTO      addr_not_reset loop_cnt
		cmd[4] = (3 << 30) | (0 << 20) | (1 << 0); // GOTO
		cmd[5] = 0; // NOP
	
	} else if (mode == 0x2) { // multi- bist write/read

		sram_st = 0;
		// sram_sp = 511;
		sram_sp = 7;
		// cmd queue
		//        op_code         start              stop        pattern    dq_inv     dm_inv    dq_rotate
		//        repeat
		cmd[0] = (1 << 30) | (sram_st << 21) | (sram_sp << 12) | (5 << 9) | (0 << 8) | (0 << 7) | (0 << 4) |
			 (0 << 0); // W  1~17  prbs  repeat0
		cmd[1] = (2 << 30) | (sram_st << 21) | (sram_sp << 12) | (5 << 9) | (0 << 8) | (0 << 7) | (0 << 4) |
			 (0 << 0); // R  1~17  prbs  repeat0
		cmd[2] = 0; // NOP
		cmd[3] = 0; // NOP
		cmd[4] = 0; // NOP
		cmd[5] = 0; // NOP
	} else if (mode == 0x12) { // with Error enject,  multi- bist write/read
		//----for Error enject simulation only
		rddata = mmio_rd32(0x0084 + PHYD_BASE_ADDR);
		rddata = modified_bits_by_value(rddata, 0x1d, 4, 0); // param_phyd_pirdlvl_trig_lvl_start
		rddata = modified_bits_by_value(rddata, 0x1f, 12, 8); // param_phyd_pirdlvl_trig_lvl_end
		mmio_wr32(0x0084 + PHYD_BASE_ADDR, rddata);
		//----for Error enject simulation only
		sram_st = 0;
		// sram_sp = 511;
		sram_sp = 7;
		// cmd queue
		//        op_code         start              stop        pattern    dq_inv     dm_inv    dq_rotate
		//        repeat
		cmd[0] = (1 << 30) | (sram_st << 21) | (sram_sp << 12) | (5 << 9) | (0 << 8) | (0 << 7) | (0 << 4) |
			 (0 << 0); // W  1~17  prbs  repeat0
		cmd[1] = (2 << 30) | (sram_st << 21) | (sram_sp << 12) | (5 << 9) | (0 << 8) | (0 << 7) | (0 << 4) |
			 (0 << 0); // R  1~17  prbs  repeat0
		cmd[2] = 0; // NOP
		cmd[3] = 0; // NOP
		cmd[4] = 0; // NOP
		cmd[5] = 0; // NOP
	} else if (mode == 0x10) { // with Error enject,  multi- bist write/read
		//----for Error enject simulation only
		rddata = mmio_rd32(0x0084 + PHYD_BASE_ADDR);
		rddata = modified_bits_by_value(rddata, 1, 4, 0); // param_phyd_pirdlvl_trig_lvl_start
		rddata = modified_bits_by_value(rddata, 3, 12, 8); // param_phyd_pirdlvl_trig_lvl_end
		mmio_wr32(0x0084 + PHYD_BASE_ADDR, rddata);
		//----for Error enject simulation only
		sram_st = 0;
		// sram_sp = 511;
		sram_sp = 3;
		// cmd queue
		//        op_code         start              stop        pattern    dq_inv     dm_inv    dq_rotate
		//        repeat
		// cmd[0] = (1 << 30) | (sram_st << 21) | (sram_sp << 12) | (5 << 9) | (0 << 8) | (0 << 7) | (0 << 4) |
		//          (0 << 0);  // W  1~17  prbs  repeat0
		cmd[0] = (2 << 30) | (sram_st << 21) | (sram_sp << 12) | (5 << 9) | (0 << 8) | (0 << 7) | (0 << 4) |
			 (0 << 0); // R  1~17  prbs  repeat0
		// cmd[1] = (2 << 30) | (sram_st << 21) | (sram_sp << 12) | (5 << 9) | (0 << 8) | (0 << 7) | (0 << 4) |
		//          (0 << 0);  // R  1~17  prbs  repeat0
		cmd[1] = 0; //NOP
		cmd[2] = 0; // NOP
		cmd[3] = 0; // NOP
		cmd[4] = 0; // NOP
		cmd[5] = 0; // NOP
	}
	// write cmd queue
	for (i = 0; i < 6; i = i + 1) {
		mmio_wr32(DDR_BIST_BASE + 0x40 + i * 4, cmd[i]);
	};
	// specified DDR space
	mmio_wr32(DDR_BIST_BASE + 0x10, 0x00000000);
	mmio_wr32(DDR_BIST_BASE + 0x14, 0x000fffff);
#ifdef X16_MODE
	// specified AXI address step
	mmio_wr32(DDR_BIST_BASE + 0x18, 0x00000004);
#else
	// specified AXI address step
	mmio_wr32(DDR_BIST_BASE + 0x18, 0x00000008);
#endif
	uartlog("%s\n", __func__);
	KC_MSG("%s\n", __func__);
}

void cvx16_bist_wdqlvl_init(uint32_t mode)
{
	uint32_t cmd[6];
	uint32_t sram_st;
	uint32_t sram_sp;
	uint32_t fmax;
	uint32_t fmin;
	// uint32_t wdqlvl_vref_start; //unused
	// uint32_t wdqlvl_vref_end; //unused
	// uint32_t wdqlvl_vref_step; //unused
	int i;

	uartlog("%s\n", __func__);
	ddr_debug_wr32(0x233);
	ddr_debug_num_write();
	KC_MSG("%s mode = %x\n", __func__, mode);

	// bist clock enable
	mmio_wr32(DDR_BIST_BASE + 0x0, 0x00060006);
	if (mode == 0x0) { // phyd pattern
		sram_st = 0;
		// sram_sp = 511;
		sram_sp = 3;
		// cmd queue
		//        op_code         start              stop        pattern    dq_inv     dm_inv    dq_rotate
		//        repeat
		cmd[0] = (1 << 30) | (sram_st << 21) | (sram_sp << 12) | (5 << 9) | (0 << 8) | (0 << 7) | (0 << 4) |
			 (0 << 0); // W  1~17  prbs  repeat0
		cmd[1] = (2 << 30) | (sram_st << 21) | (sram_sp << 12) | (5 << 9) | (0 << 8) | (0 << 7) | (0 << 4) |
			 (0 << 0); // R  1~17  prbs  repeat0
		cmd[2] = 0; // NOP
		cmd[3] = 0; // NOP
		cmd[4] = 0; // NOP
		cmd[5] = 0; // NOP
	} else if (mode == 0x1) { // bist write/read
		sram_st = 0;
		fmax = 15;
		fmin = 5;
		sram_st = 0;
		sram_sp = 9 * (fmin + fmax) * (fmax - fmin + 1) / 2 / 4 + (fmax - fmin + 1); // 8*f/4 -1
		KC_MSG("sram_sp = %x\n", sram_sp);

		// bist sso_period
		mmio_wr32(DDR_BIST_BASE + 0x24, (fmax << 8) + fmin);

		// cmd queue
		//        op_code         start              stop        pattern    dq_inv     dm_inv    dq_rotate
		//        repeat
		cmd[0] = (1 << 30) | (sram_st << 21) | (511 << 12) | (5 << 9) | (0 << 8) | (0 << 7) | (0 << 4) |
			 (2 << 0); // W  1~17  prbs  repeat0
		cmd[1] = (2 << 30) | (sram_st << 21) | (511 << 12) | (5 << 9) | (0 << 8) | (0 << 7) | (0 << 4) |
			 (2 << 0); // R  1~17  prbs  repeat0
		cmd[2] = (1 << 30) | (sram_st << 21) | (sram_sp << 12) | (6 << 9) | (0 << 8) | (0 << 7) | (0 << 4) |
			 (2 << 0); // W  1~17  sram  repeat0
		cmd[3] = (2 << 30) | (sram_st << 21) | (sram_sp << 12) | (6 << 9) | (0 << 8) | (0 << 7) | (0 << 4) |
			 (2 << 0); // R  1~17  sram  repeat0
		//       GOTO      addr_not_reset loop_cnt
		cmd[4] = (3 << 30) | (0 << 20) | (1 << 0); // GOTO
		cmd[5] = 0; // NOP

	} else if (mode == 0x11) { // bist write/read
		//----for Error enject simulation only
		rddata = 0x00000000;
		rddata = modified_bits_by_value(rddata, 0x1, 6, 0); // param_phyd_dfi_wdqlvl_vref_start
		rddata = modified_bits_by_value(rddata, 0x3, 14, 8); // param_phyd_dfi_wdqlvl_vref_end
		rddata = modified_bits_by_value(rddata, 0x1, 19, 16); // param_phyd_dfi_wdqlvl_vref_step
		mmio_wr32(0x0190 + PHYD_BASE_ADDR, rddata);
		//----for Error enject simulation only
		sram_st = 0;
		// sram_sp = 511;
		sram_sp = 3;
		// cmd queue
		//        op_code         start              stop        pattern    dq_inv     dm_inv    dq_rotate
		//        repeat
		cmd[0] = (1 << 30) | (sram_st << 21) | (sram_sp << 12) | (5 << 9) | (0 << 8) | (0 << 7) | (0 << 4) |
			 (0 << 0); // W  1~17  prbs  repeat0
		cmd[1] = (2 << 30) | (sram_st << 21) | (sram_sp << 12) | (5 << 9) | (0 << 8) | (0 << 7) | (0 << 4) |
			 (0 << 0); // R  1~17  prbs  repeat0
		cmd[2] = 0; // NOP
		cmd[3] = 0; // NOP
		cmd[4] = 0; // NOP
		cmd[5] = 0; // NOP
	} else if (mode == 0x12) { // bist write/read
		//----for Error enject simulation only
		rddata = 0x00000000;
		rddata = modified_bits_by_value(rddata, 0x1d, 6, 0); // param_phyd_dfi_wdqlvl_vref_start
		rddata = modified_bits_by_value(rddata, 0x1f, 14, 8); // param_phyd_dfi_wdqlvl_vref_end
		rddata = modified_bits_by_value(rddata, 0x1, 19, 16); // param_phyd_dfi_wdqlvl_vref_step
		mmio_wr32(0x0190 + PHYD_BASE_ADDR, rddata);
		//----for Error enject simulation only
		sram_st = 0;
		// sram_sp = 511;
		sram_sp = 3;
		// cmd queue
		//        op_code         start              stop        pattern    dq_inv     dm_inv    dq_rotate
		//        repeat
		cmd[0] = (1 << 30) | (sram_st << 21) | (sram_sp << 12) | (5 << 9) | (0 << 8) | (0 << 7) | (0 << 4) |
			 (0 << 0); // W  1~17  prbs  repeat0
		cmd[1] = (2 << 30) | (sram_st << 21) | (sram_sp << 12) | (5 << 9) | (0 << 8) | (0 << 7) | (0 << 4) |
			 (0 << 0); // R  1~17  prbs  repeat0
		cmd[2] = 0; // NOP
		cmd[3] = 0; // NOP
		cmd[4] = 0; // NOP
		cmd[5] = 0; // NOP
	} else {
		sram_st = 0;
		// sram_sp = 511;
		sram_sp = 3;
		// cmd queue
		//        op_code         start              stop        pattern    dq_inv     dm_inv    dq_rotate
		//        repeat
		cmd[0] = (1 << 30) | (sram_st << 21) | (sram_sp << 12) | (5 << 9) | (0 << 8) | (0 << 7) | (0 << 4) |
			 (0 << 0); // W  1~17  prbs  repeat0
		cmd[1] = (2 << 30) | (sram_st << 21) | (sram_sp << 12) | (5 << 9) | (0 << 8) | (0 << 7) | (0 << 4) |
			 (0 << 0); // R  1~17  prbs  repeat0
		cmd[2] = 0; // NOP
		cmd[3] = 0; // NOP
		cmd[4] = 0; // NOP
		cmd[5] = 0; // NOP
	}
	// write cmd queue
	for (i = 0; i < 6; i = i + 1) {
		mmio_wr32(DDR_BIST_BASE + 0x40 + i * 4, cmd[i]);
	};
	// specified DDR space
	mmio_wr32(DDR_BIST_BASE + 0x10, 0x00000000);
	mmio_wr32(DDR_BIST_BASE + 0x14, 0x000fffff);
#ifdef X16_MODE
	// specified AXI address step
	mmio_wr32(DDR_BIST_BASE + 0x18, 0x00000004);
#else
	// specified AXI address step
	mmio_wr32(DDR_BIST_BASE + 0x18, 0x00000008);
#endif
	uartlog("%s done\n", __func__);
	KC_MSG("%s done\n", __func__);
}

void cvx16_bist_wdmlvl_init(void)
{
	uint32_t cmd[6];
	uint32_t sram_st;
	uint32_t sram_sp;
	uint32_t fmax;
	uint32_t fmin;
	int i;

	uartlog("%s\n", __func__);
	ddr_debug_wr32(0x233);
	ddr_debug_num_write();
	KC_MSG("%s\n", __func__);

	// bist clock enable
	mmio_wr32(DDR_BIST_BASE + 0x0, 0x00060006);
	sram_st = 0;
	fmax = 15;
	fmin = 5;
	sram_st = 0;
	sram_sp = 9 * (fmin + fmax) * (fmax - fmin + 1) / 2 / 4 + (fmax - fmin + 1); // 8*f/4 -1
	KC_MSG("sram_sp = %x\n", sram_sp);

	// bist sso_period
	mmio_wr32(DDR_BIST_BASE + 0x24, (fmax << 8) + fmin);
	// cmd queue
	//        op_code         start              stop        pattern    dq_inv     dm_inv    dq_rotate
	//        repeat
	cmd[0] = (1 << 30) | (sram_st << 21) | (sram_sp << 12) | (3 << 9) | (0 << 8) | (0 << 7) | (0 << 4) |
		 (0 << 0); // bist write pat 3 = 0x0f
	cmd[1] = (1 << 30) | (sram_st << 21) | (sram_sp << 12) | (7 << 9) | (0 << 8) | (0 << 7) | (0 << 4) |
		 (0 << 0); // bist write pat 7 = sso_8x1 DM toggle verison
	cmd[2] = (2 << 30) | (sram_st << 21) | (sram_sp << 12) | (7 << 9) | (0 << 8) | (0 << 7) | (0 << 4) |
		 (0 << 0); // bist read pat 7
	cmd[3] = 0; // NOP
	cmd[4] = 0; // NOP
	cmd[5] = 0; // NOP
	// write cmd queue
	for (i = 0; i < 6; i = i + 1) {
		mmio_wr32(DDR_BIST_BASE + 0x40 + i * 4, cmd[i]);
	};
	// specified DDR space
	mmio_wr32(DDR_BIST_BASE + 0x10, 0x00000000);
	mmio_wr32(DDR_BIST_BASE + 0x14, 0x000fffff);
#ifdef X16_MODE
	// specified AXI address step
	mmio_wr32(DDR_BIST_BASE + 0x18, 0x00000004);
#else
	// specified AXI address step
	mmio_wr32(DDR_BIST_BASE + 0x18, 0x00000008);
#endif
	uartlog("%s done\n", __func__);
	KC_MSG("%s done\n", __func__);
}

void cvx16_bist_start_check(uint32_t *bist_result, uint64_t *err_data_odd, uint64_t *err_data_even)
{
	uint64_t err_data_even_l;
	uint64_t err_data_even_h;
	uint64_t err_data_odd_l;
	uint64_t err_data_odd_h;

	uartlog("%s\n", __func__);
	ddr_debug_wr32(0x25);
	ddr_debug_num_write();
#ifdef X16_MODE
	// bist enable
	mmio_wr32(DDR_BIST_BASE + 0x0, 0x00030003);
	uartlog("bist start\n");
	KC_MSG("bist start\n");

#else
	// bist enable
	mmio_wr32(DDR_BIST_BASE + 0x0, 0x00010001);
	uartlog("bist start\n");
	KC_MSG("bist start\n");

#endif
	// polling bist done
	//    while (get_bits_from_value(2)  == 0x0 = mmio_rd32(DDR_BIST_BASE + 0x80),2);
	while (1) {
		rddata = mmio_rd32(DDR_BIST_BASE + 0x80);
		if (get_bits_from_value(rddata, 2, 2) == 1) {
			break;
		}
	}
	uartlog("bist done\n");
	KC_MSG("Read bist done %x ...\n", rddata);

	if (get_bits_from_value(rddata, 3, 3) == 1) {
		// opdelay(10);
		*bist_result = 0;
		uartlog("bist fail\n");
		uartlog("err data\n");
		// read err_data
		err_data_odd_l = mmio_rd32(DDR_BIST_BASE + 0x88);
		err_data_odd_h = mmio_rd32(DDR_BIST_BASE + 0x8c);
		err_data_even_l = mmio_rd32(DDR_BIST_BASE + 0x90);
		err_data_even_h = mmio_rd32(DDR_BIST_BASE + 0x94);
		*err_data_odd = err_data_odd_h << 32 | err_data_odd_l;
		*err_data_even = err_data_even_h << 32 | err_data_even_l;
	} else {
		// opdelay(10);
		*bist_result = 1;
		uartlog("bist pass\n");
		*err_data_odd = 0;
		*err_data_even = 0;
	}
	// bist disable
	mmio_wr32(DDR_BIST_BASE + 0x0, 0x00050000);
}


void cvx16_synp_mrw(uint32_t addr, uint32_t data)
{
	uint32_t init_dis_auto_zq;

	uartlog("%s\n", __func__);
	ddr_debug_wr32(0x21);
	ddr_debug_num_write();
	// ZQCTL0.dis_auto_zq to 1.
	rddata = mmio_rd32(cfg_base + 0x180);
	// if (rddata[31] == 0b0) {
	if (get_bits_from_value(rddata, 31, 31) == 0) {
		init_dis_auto_zq = 0;
		// rddata[31] = 1;
		rddata = modified_bits_by_value(rddata, 1, 31, 31);
		mmio_wr32(cfg_base + 0x180, rddata);
		uartlog("non-lp4 Write ZQCTL0.dis_auto_zq to 1\n");
		// opdelay(256);
		uartlog("Wait tzqcs = 128 cycles\n");
	} else {
		init_dis_auto_zq = 1;
	}
	// Poll MRSTAT.mr_wr_busy until it is 0
	uartlog("Poll MRSTAT.mr_wr_busy until it is 0\n");
	rddata = 0;
	while (1) {
		rddata = mmio_rd32(cfg_base + 0x18);
		if (get_bits_from_value(rddata, 0, 0) == 0) {
			break;
		}
	}
	uartlog("non-lp4 Poll MRSTAT.mr_wr_busy finish\n");
	// Write the MRCTRL0.mr_type, MRCTRL0.mr_addr, MRCTRL0.mr_rank and (for MRWs) MRCTRL1.mr_data
	// rddata[31:0]  = 0;
	// rddata[0]     = 0;       // mr_type  0:write   1:read
	// rddata[5:4]   = 1;       // mr_rank
	// rddata[15:12] = addr;    // mr_addr
	rddata = modified_bits_by_value(rddata, 0, 31, 0);
	rddata = modified_bits_by_value(rddata, 0, 0, 0);
	rddata = modified_bits_by_value(rddata, 1, 5, 4);
	rddata = modified_bits_by_value(rddata, addr, 15, 12);
	mmio_wr32(cfg_base + 0x10, rddata);
	uartlog("non-lp4 Write the MRCTRL0\n");
	// rddata[31:0] = 0;
	// rddata[15:0] = data;     // mr_data
	rddata = modified_bits_by_value(rddata, 0, 31, 0);
	rddata = modified_bits_by_value(rddata, data, 31, 0);
	mmio_wr32(cfg_base + 0x14, rddata);
	uartlog("non-lp4 Write the MRCTRL1\n");
	// Write MRCTRL0.mr_wr to 1
	rddata = mmio_rd32(cfg_base + 0x10);
	// rddata[31] = 1;
	rddata = modified_bits_by_value(rddata, 1, 31, 31);
	mmio_wr32(cfg_base + 0x10, rddata);
	uartlog("non-lp4 Write MRCTRL0.mr_wr to 1\n");
	if (init_dis_auto_zq == 0) {
		// ZQCTL0.dis_auto_zq to 0.
		rddata = mmio_rd32(cfg_base + 0x180);
		// rddata[31] = 0;
		rddata = modified_bits_by_value(rddata, 0, 31, 31);
		mmio_wr32(cfg_base + 0x180, rddata);
		uartlog("non-lp4 Write ZQCTL0.dis_auto_zq to 0\n");
	}
}

void cvx16_chg_pll_freq(void)
{
	uint32_t EN_PLL_SPEED_CHG;
	uint32_t CUR_PLL_SPEED;
	uint32_t NEXT_PLL_SPEED;

	uartlog("%s\n", __func__);
	ddr_debug_wr32(0x04);
	ddr_debug_num_write();
	// Change PLL frequency
	// TOP_REG_RESETZ_DIV =0
	rddata = 0x00000000;
	mmio_wr32(0x04 + CV_DDR_PHYD_APB, rddata);
	// TOP_REG_RESETZ_DQS =0
	mmio_wr32(0x08 + CV_DDR_PHYD_APB, rddata);
	// TOP_REG_DDRPLL_MAS_RSTZ_DIV  =0
	rddata = mmio_rd32(0x0C + CV_DDR_PHYD_APB);
	rddata = modified_bits_by_value(rddata, 0, 7, 7);
	mmio_wr32(0x0C + CV_DDR_PHYD_APB, rddata);
	uartlog("RSTZ_DIV=0\n");
	rddata = mmio_rd32(0x4c + CV_DDR_PHYD_APB);
	rddata = mmio_rd32(0x4c + CV_DDR_PHYD_APB);
	rddata = mmio_rd32(0x4c + CV_DDR_PHYD_APB);
	rddata = mmio_rd32(0x4c + CV_DDR_PHYD_APB);
	rddata = mmio_rd32(0x4c + CV_DDR_PHYD_APB);
	rddata = mmio_rd32(0x4c + CV_DDR_PHYD_APB);
	EN_PLL_SPEED_CHG = get_bits_from_value(rddata, 0, 0);
	CUR_PLL_SPEED = get_bits_from_value(rddata, 5, 4);
	NEXT_PLL_SPEED = get_bits_from_value(rddata, 9, 8);
	KC_MSG("CUR_PLL_SPEED = %x, NEXT_PLL_SPEED = %x, EN_PLL_SPEED_CHG=%x\n", CUR_PLL_SPEED, NEXT_PLL_SPEED,
	       EN_PLL_SPEED_CHG);

	if (EN_PLL_SPEED_CHG) {
		if (NEXT_PLL_SPEED == 0) { // next clk_div40
			rddata = modified_bits_by_value(rddata, NEXT_PLL_SPEED, 5, 4);
			rddata = modified_bits_by_value(rddata, CUR_PLL_SPEED, 9, 8);
			mmio_wr32(0x4c + CV_DDR_PHYD_APB, rddata);
			cvx16_clk_div40();
			uartlog("clk_div40\n");
			KC_MSG("clk_div40\n");
		} else {
			if (NEXT_PLL_SPEED == 0x2) { // next clk normal
				rddata = modified_bits_by_value(rddata, NEXT_PLL_SPEED, 5, 4);
				rddata = modified_bits_by_value(rddata, CUR_PLL_SPEED, 9, 8);
				mmio_wr32(0x4c + CV_DDR_PHYD_APB, rddata);
				cvx16_clk_normal();
				uartlog("clk_normal\n");
				KC_MSG("clk_normal\n");
			} else {
				if (NEXT_PLL_SPEED == 0x1) { // next clk normal div_2
					rddata = modified_bits_by_value(rddata, NEXT_PLL_SPEED, 5, 4);
					rddata = modified_bits_by_value(rddata, CUR_PLL_SPEED, 9, 8);
					mmio_wr32(0x4c + CV_DDR_PHYD_APB, rddata);
					cvx16_clk_div2();
					uartlog("clk_div2\n");
					KC_MSG("clk_div2\n");
				}
			}
		}
		//         opdelay(100000);  //  1000ns
	}
	// TOP_REG_RESETZ_DIV  =1
	rddata = 0x00000001;
	mmio_wr32(0x04 + CV_DDR_PHYD_APB, rddata);
	rddata = mmio_rd32(0x0C + CV_DDR_PHYD_APB);
	// rddata[7]   = 1;    //TOP_REG_DDRPLL_MAS_RSTZ_DIV
	rddata = modified_bits_by_value(rddata, 1, 7, 7);
	mmio_wr32(0x0C + CV_DDR_PHYD_APB, rddata);
	uartlog("RSTZ_DIV=1\n");
	// rddata[0]   = 1;    //TOP_REG_RESETZ_DQS
	rddata = 0x00000001;
	mmio_wr32(0x08 + CV_DDR_PHYD_APB, rddata);
	uartlog("TOP_REG_RESETZ_DQS\n");
	KC_MSG("Wait for DRRPLL_SLV_LOCK=1...\n");

#ifdef REAL_LOCK
	rddata = modified_bits_by_value(rddata, 0, 15, 15);
	while (get_bits_from_value(rddata, 15, 15) == 0) {
		rddata = mmio_rd32(0x10 + CV_DDR_PHYD_APB);
		KC_MSG("REAL_LOCK.\n");

		opdelay(200);
	}
#else
	KC_MSG("check PLL lock...  pll init\n");

#endif
	//} Change PLL frequency
}

void cvx16_dll_cal(void)
{
	uint32_t EN_PLL_SPEED_CHG;
	uint32_t CUR_PLL_SPEED;
	uint32_t NEXT_PLL_SPEED;

	KC_MSG("Do DLLCAL ...\n");

	uartlog("%s\n", __func__);
	ddr_debug_wr32(0x2b);
	ddr_debug_num_write();
	// TOP_REG_EN_PLL_SPEED_CHG
	//     <= #RD (~pwstrb_mask[0] & TOP_REG_EN_PLL_SPEED_CHG) |  pwstrb_mask_pwdata[0];
	// TOP_REG_CUR_PLL_SPEED   [1:0]
	//     <= #RD (~pwstrb_mask[5:4] & TOP_REG_CUR_PLL_SPEED[1:0]) |  pwstrb_mask_pwdata[5:4];
	// TOP_REG_NEXT_PLL_SPEED  [1:0]
	//     <= #RD (~pwstrb_mask[9:8] & TOP_REG_NEXT_PLL_SPEED[1:0]) |  pwstrb_mask_pwdata[9:8];
	rddata = mmio_rd32(0x4c + CV_DDR_PHYD_APB);
	EN_PLL_SPEED_CHG = get_bits_from_value(rddata, 0, 0);
	CUR_PLL_SPEED = get_bits_from_value(rddata, 5, 4);
	NEXT_PLL_SPEED = get_bits_from_value(rddata, 9, 8);
	KC_MSG("CUR_PLL_SPEED = %x, NEXT_PLL_SPEED = %x, EN_PLL_SPEED_CHG=%x\n", CUR_PLL_SPEED, NEXT_PLL_SPEED,
	       EN_PLL_SPEED_CHG);

	if (CUR_PLL_SPEED != 0) { // only do calibration and update when high speed
		// param_phyd_dll_rx_start_cal <= int_regin[1];
		// param_phyd_dll_tx_start_cal <= int_regin[17];
		rddata = mmio_rd32(0x0040 + PHYD_BASE_ADDR);
		rddata = modified_bits_by_value(rddata, 0, 1, 1);
		rddata = modified_bits_by_value(rddata, 0, 17, 17);
		mmio_wr32(0x0040 + PHYD_BASE_ADDR, rddata);
		// param_phyd_dll_rx_start_cal <= int_regin[1];
		// param_phyd_dll_tx_start_cal <= int_regin[17];
		rddata = mmio_rd32(0x0040 + PHYD_BASE_ADDR);
		rddata = modified_bits_by_value(rddata, 1, 1, 1);
		rddata = modified_bits_by_value(rddata, 1, 17, 17);
		mmio_wr32(0x0040 + PHYD_BASE_ADDR, rddata);
		rddata = 0x00000000;
		while (get_bits_from_value(rddata, 16, 16) == 0) {
			rddata = mmio_rd32(0x3014 + PHYD_BASE_ADDR);
		}
		KC_MSG("DLL lock !\n");

		uartlog("DLL lock\n");
		// opdelay(1000);
		uartlog("Do DLLUPD\n");
		// cvx16_dll_cal_status();
	} else { // stop calibration and update when low speed
		// param_phyd_dll_rx_start_cal <= int_regin[1];
		// param_phyd_dll_tx_start_cal <= int_regin[17];
		rddata = mmio_rd32(0x0040 + PHYD_BASE_ADDR);
		rddata = modified_bits_by_value(rddata, 0, 1, 1);
		rddata = modified_bits_by_value(rddata, 0, 17, 17);
		mmio_wr32(0x0040 + PHYD_BASE_ADDR, rddata);
	}
	KC_MSG("Do DLLCAL Finish\n");

	uartlog("Do DLLCAL Finish\n");
}

void cvx16_dll_cal_phya_enautok(void)
{
	KC_MSG("Do DLLCAL enautok ...\n");

	uartlog("cvx16_dll_cal enautok\n");
	ddr_debug_wr32(0x2b);
	ddr_debug_num_write();
	KC_MSG("DLLCAL enautok\n");

	// param_phyd_dll_rx_sw_mode    [0]
	// param_phyd_dll_rx_start_cal  [1]
	// param_phyd_dll_rx_cntr_mode  [2]
	// param_phyd_dll_rx_hwrst_time [3]
	// param_phyd_dll_tx_sw_mode    [16]
	// param_phyd_dll_tx_start_cal  [17]
	// param_phyd_dll_tx_cntr_mode  [18]
	// param_phyd_dll_tx_hwrst_time [19]
	rddata = mmio_rd32(0x0040 + PHYD_BASE_ADDR);
	rddata = modified_bits_by_value(rddata, 0b0000, 3, 0);
	rddata = modified_bits_by_value(rddata, 0b0000, 19, 16);
	mmio_wr32(0x0040 + PHYD_BASE_ADDR, rddata);
	// param_phya_reg_rx_ddrdll_enautok [0]
	// param_phya_reg_tx_ddrdll_enautok [16]
	rddata = mmio_rd32(0x0140 + PHYD_BASE_ADDR);
	rddata = modified_bits_by_value(rddata, 1, 0, 0);
	rddata = modified_bits_by_value(rddata, 1, 16, 16);
	mmio_wr32(0x0140 + PHYD_BASE_ADDR, rddata);
	while (1) {
		rddata = mmio_rd32(0x3014 + PHYD_BASE_ADDR);
		if ((get_bits_from_value(rddata, 16, 16) & get_bits_from_value(rddata, 0, 0)) != 0) {
			break;
		}
	}
	KC_MSG("DLL lock !\n");

	uartlog("DLL lock\n");
	// opdelay(1000);
	uartlog("Do DLLUPD\n");
	// cvx16_dll_cal_status();
	// param_phyd_dll_rx_start_cal <= int_regin[1];
	// param_phyd_dll_tx_start_cal <= int_regin[17];
	rddata = mmio_rd32(0x0040 + PHYD_BASE_ADDR);
	rddata = modified_bits_by_value(rddata, 0, 1, 1);
	rddata = modified_bits_by_value(rddata, 0, 17, 17);
	mmio_wr32(0x0040 + PHYD_BASE_ADDR, rddata);
	KC_MSG("Do DLLCAL enautok\n");

	uartlog("Do DLLCAL enautok Finish\n");
}

void cvx16_clk_normal(void)
{
	uartlog("%s\n", __func__);
	ddr_debug_wr32(0x03);
	ddr_debug_num_write();
	KC_MSG("back to original frequency !!!\n\n");

	rddata = mmio_rd32(0x0c + CV_DDR_PHYD_APB);
	// rddata[13] TOP_REG_DDRPLL_SEL_LOW_SPEED 0
	// rddata[14] TOP_REG_DDRPLL_MAS_DIV_OUT_SEL 0
	rddata = modified_bits_by_value(rddata, 0, 13, 13);
	rddata = modified_bits_by_value(rddata, 0, 14, 14);
	mmio_wr32(0x0c + CV_DDR_PHYD_APB, rddata);
#ifdef SSC_EN
	//==============================================================
	// Enable SSC
	//==============================================================
	rddata = reg_set; // TOP_REG_SSC_SET
	mmio_wr32(0x54 + 0x03002900, rddata);
	rddata = get_bits_from_value(reg_span, 15, 0); // TOP_REG_SSC_SPAN
	mmio_wr32(0x58 + 0x03002900, rddata);
	rddata = get_bits_from_value(reg_step, 23, 0); // TOP_REG_SSC_STEP
	mmio_wr32(0x5C + 0x03002900, rddata);
	rddata = mmio_rd32(0x50 + 0x03002900);
	rddata = modified_bits_by_value(rddata, ~get_bits_from_value(rddata, 0, 0), 0, 0); // TOP_REG_SSC_SW_UP
	rddata = modified_bits_by_value(rddata, 1, 1, 1); // TOP_REG_SSC_EN_SSC
	rddata = modified_bits_by_value(rddata, 0, 3, 2); // TOP_REG_SSC_SSC_MODE
	rddata = modified_bits_by_value(rddata, 0, 4, 4); // TOP_REG_SSC_BYPASS
	rddata = modified_bits_by_value(rddata, 1, 5, 5); // extpulse
	rddata = modified_bits_by_value(rddata, 0, 6, 6); // ssc_syn_fix_div
	mmio_wr32(0x50 + 0x03002900, rddata);
	uartlog("SSC_EN\n");
#else
#ifdef SSC_BYPASS
	rddata = (reg_set & 0xfc000000) + 0x04000000; // TOP_REG_SSC_SET
	mmio_wr32(0x54 + 0x03002900, rddata);
	rddata = get_bits_from_value(reg_span, 15, 0); // TOP_REG_SSC_SPAN
	mmio_wr32(0x58 + 0x03002900, rddata);
	rddata = get_bits_from_value(reg_step, 23, 0); // TOP_REG_SSC_STEP
	mmio_wr32(0x5C + 0x03002900, rddata);
	rddata = mmio_rd32(0x50 + 0x03002900);
	rddata = modified_bits_by_value(rddata, ~get_bits_from_value(rddata, 0, 0), 0, 0); // TOP_REG_SSC_SW_UP
	rddata = modified_bits_by_value(rddata, 0, 1, 1); // TOP_REG_SSC_EN_SSC
	rddata = modified_bits_by_value(rddata, 0, 3, 2); // TOP_REG_SSC_SSC_MODE
	rddata = modified_bits_by_value(rddata, 0, 4, 4); // TOP_REG_SSC_BYPASS
	rddata = modified_bits_by_value(rddata, 1, 5, 5); // TOP_REG_SSC_EXTPULSE
	rddata = modified_bits_by_value(rddata, 1, 6, 6); // ssc_syn_fix_div
	uartlog("SSC_BYPASS\n");
#else
	//==============================================================
	// SSC_EN =0
	//==============================================================
	uartlog("SSC_EN =0\n");
	rddata = reg_set; // TOP_REG_SSC_SET
	mmio_wr32(0x54 + 0x03002900, rddata);
	rddata = get_bits_from_value(reg_span, 15, 0); // TOP_REG_SSC_SPAN
	mmio_wr32(0x58 + 0x03002900, rddata);
	rddata = get_bits_from_value(reg_step, 23, 0); // TOP_REG_SSC_STEP
	mmio_wr32(0x5C + 0x03002900, rddata);
	rddata = mmio_rd32(0x50 + 0x03002900);
	rddata = modified_bits_by_value(rddata, ~get_bits_from_value(rddata, 0, 0), 0, 0); // TOP_REG_SSC_SW_UP
	rddata = modified_bits_by_value(rddata, 0, 1, 1); // TOP_REG_SSC_EN_SSC
	rddata = modified_bits_by_value(rddata, 0, 3, 2); // TOP_REG_SSC_SSC_MODE
	rddata = modified_bits_by_value(rddata, 0, 4, 4); // TOP_REG_SSC_BYPASS
	rddata = modified_bits_by_value(rddata, 1, 5, 5); // TOP_REG_SSC_EXTPULSE
	rddata = modified_bits_by_value(rddata, 0, 6, 6); // ssc_syn_fix_div
	mmio_wr32(0x50 + 0x03002900, rddata);
	uartlog("SSC_OFF\n");
#endif // SSC_BYPASS
#endif // SSC_EN
	uartlog("back to original frequency\n");
}

void cvx16_clk_div2(void)
{
	uartlog("%s\n", __func__);
	ddr_debug_wr32(0x01);
	ddr_debug_num_write();
	KC_MSG("div2 original frequency !!!\n\n");

	rddata = mmio_rd32(0x0c + CV_DDR_PHYD_APB);
	// rddata[14] = 1  ;  // TOP_REG_DDRPLL_MAS_DIV_OUT_SEL 1
	rddata = modified_bits_by_value(rddata, 1, 14, 14);
	mmio_wr32(0x0c + CV_DDR_PHYD_APB, rddata);
	uartlog("div2 original frequency\n");
}

void cvx16_INT_ISR_08(void)
{
	uint32_t EN_PLL_SPEED_CHG;
	uint32_t CUR_PLL_SPEED;
	uint32_t NEXT_PLL_SPEED;

	uartlog("%s\n", __func__);
	ddr_debug_wr32(0x1c);
	ddr_debug_num_write();
	// param_phyd_clkctrl_init_complete   <= int_regin[0];
	rddata = 0x00000000;
	mmio_wr32(0x0118 + PHYD_BASE_ADDR, rddata);
	//----------------------------------------------------
	rddata = mmio_rd32(0x4c + CV_DDR_PHYD_APB);
	EN_PLL_SPEED_CHG = get_bits_from_value(rddata, 0, 0);
	CUR_PLL_SPEED = get_bits_from_value(rddata, 5, 4);
	NEXT_PLL_SPEED = get_bits_from_value(rddata, 9, 8);
	KC_MSG("CUR_PLL_SPEED = %x, NEXT_PLL_SPEED = %x, EN_PLL_SPEED_CHG=%x\n", CUR_PLL_SPEED, NEXT_PLL_SPEED,
	       EN_PLL_SPEED_CHG);

	//----------------------------------------------------
	cvx16_ddr_phy_power_on_seq2();
	cvx16_set_dfi_init_complete();
}

void cvx16_clk_div40(void)
{
	uartlog("%s\n", __func__);
	ddr_debug_wr32(0x02);
	ddr_debug_num_write();
	KC_MSG("Enter low D40 frequency !!!\n\n");

	rddata = mmio_rd32(0x0c + CV_DDR_PHYD_APB);
	// TOP_REG_DDRPLL_SEL_LOW_SPEED =1
	rddata = modified_bits_by_value(rddata, 1, 13, 13);
	mmio_wr32(0x0c + CV_DDR_PHYD_APB, rddata);
	uartlog("Enter low D40 frequency\n");
}

void cvx16_ddr_phy_power_on_seq1(void)
{
	// power_seq_1
	uartlog("%s\n", __func__);
	ddr_debug_wr32(0x0e);
	ddr_debug_num_write();
	// RESETZ/CKE PD=0
	rddata = mmio_rd32(0x40 + CV_DDR_PHYD_APB);
	// TOP_REG_TX_CA_PD_CKE0
	rddata = modified_bits_by_value(rddata, 0, 24, 24);
	// TOP_REG_TX_CA_PD_RESETZ
	rddata = modified_bits_by_value(rddata, 0, 30, 30);
	mmio_wr32(0x40 + CV_DDR_PHYD_APB, rddata);
	KC_MSG("RESET PD !!!\n");

	// CA PD=0
	// All PHYA CA PD=0
	rddata = mmio_rd32(0x40 + CV_DDR_PHYD_APB);
	rddata = modified_bits_by_value(rddata, 0, 31, 0);
	mmio_wr32(0x40 + CV_DDR_PHYD_APB, rddata);
	KC_MSG("All PHYA CA PD=0 ...\n");

	// TOP_REG_TX_SEL_GPIO = 1 (DQ)
	rddata = mmio_rd32(0x1c + CV_DDR_PHYD_APB);
	rddata = modified_bits_by_value(rddata, 1, 7, 7);
	mmio_wr32(0x1c + CV_DDR_PHYD_APB, rddata);
	KC_MSG("TOP_REG_TX_SEL_GPIO = 1\n");

	// DQ PD=0
	// TOP_REG_TX_BYTE0_PD
	// TOP_REG_TX_BYTE1_PD
	rddata = 0x00000000;
	mmio_wr32(0x00 + CV_DDR_PHYD_APB, rddata);
	KC_MSG("TX_BYTE PD=0 ...\n");

	// TOP_REG_TX_SEL_GPIO = 0 (DQ)
	rddata = mmio_rd32(0x1c + CV_DDR_PHYD_APB);
	rddata = modified_bits_by_value(rddata, 0, 7, 7);
	mmio_wr32(0x1c + CV_DDR_PHYD_APB, rddata);
	KC_MSG("TOP_REG_TX_SEL_GPIO = 0\n");

	// power_seq_1
}

void cvx16_ddr_phy_power_on_seq2(void)
{
	uartlog("%s\n", __func__);
	ddr_debug_wr32(0x0f);
	ddr_debug_num_write();
	// Change PLL frequency
	KC_MSG("Change PLL frequency if necessary ...\n");

	cvx16_chg_pll_freq();
	// OEN
	// param_phyd_sel_cke_oenz        <= `PI_SD int_regin[0];
	rddata = mmio_rd32(0x0154 + PHYD_BASE_ADDR);
	rddata = modified_bits_by_value(rddata, 0, 0, 0);
	mmio_wr32(0x0154 + PHYD_BASE_ADDR, rddata);
	// param_phyd_tx_ca_oenz          <= `PI_SD int_regin[0];
	// param_phyd_tx_ca_clk0_oenz     <= `PI_SD int_regin[8];
	// param_phyd_tx_ca_clk1_oenz     <= `PI_SD int_regin[16];
	rddata = 0x00000000;
	mmio_wr32(0x0130 + PHYD_BASE_ADDR, rddata);
	// Do DLLCAL if necessary
	KC_MSG("Do DLLCAL if necessary ...\n");

	cvx16_dll_cal();
	KC_MSG("Do DLLCAL done\n");

	//    KC_MSG("Do ZQCAL if necessary ...\n");

	// cvx16_ddr_zqcal_hw_isr8(0x7);//zqcal hw mode, bit0: offset_cal, bit1:pl_en, bit2:step2_en
	// KC_MSG("Do ZQCAL done\n");

	KC_MSG("cv181x without ZQ Calibration ...\n");

	// cvx16_ddr_zq240_cal();//zq240_cal
	// KC_MSG("Do cvx16_ddr_zq240_cal done\n");

	KC_MSG("cv181x without ZQ240 Calibration ...\n");

	// zq calculate variation
	//  zq_cal_var();
	KC_MSG("zq calculate variation not run\n");

	// CA PD =0
	// All PHYA CA PD=0
	rddata = 0x80000000;
	mmio_wr32(0x40 + CV_DDR_PHYD_APB, rddata);
	KC_MSG("All PHYA CA PD=0 ...\n");

	// BYTE PD =0
	rddata = 0x00000000;
	mmio_wr32(0x00 + CV_DDR_PHYD_APB, rddata);
	KC_MSG("TX_BYTE PD=0 ...\n");

	// power_on_2
}

void cvx16_ddr_phy_power_on_seq3(void)
{
	// power on 3
	uartlog("%s\n", __func__);
	ddr_debug_wr32(0x10);
	ddr_debug_num_write();
	// RESETYZ/CKE OENZ
	// param_phyd_sel_cke_oenz        <= `PI_SD int_regin[0];
	rddata = mmio_rd32(0x0154 + PHYD_BASE_ADDR);
	rddata = modified_bits_by_value(rddata, 0, 0, 0);
	mmio_wr32(0x0154 + PHYD_BASE_ADDR, rddata);
	// param_phyd_tx_ca_oenz          <= `PI_SD int_regin[0];
	// param_phyd_tx_ca_clk0_oenz     <= `PI_SD int_regin[8];
	// param_phyd_tx_ca_clk1_oenz     <= `PI_SD int_regin[16];
	rddata = 0x00000000;
	mmio_wr32(0x0130 + PHYD_BASE_ADDR, rddata);
	uartlog("[KC Info] --> ca_oenz  ca_clk_oenz !!!\n");

	// clock gated for power save
	// param_phya_reg_tx_byte0_en_extend_oenz_gated_dline <= `PI_SD int_regin[0];
	// param_phya_reg_tx_byte1_en_extend_oenz_gated_dline <= `PI_SD int_regin[1];
	// param_phya_reg_tx_byte2_en_extend_oenz_gated_dline <= `PI_SD int_regin[2];
	// param_phya_reg_tx_byte3_en_extend_oenz_gated_dline <= `PI_SD int_regin[3];
	rddata = mmio_rd32(0x0204 + PHYD_BASE_ADDR);
	rddata = modified_bits_by_value(rddata, 1, 18, 18);
	mmio_wr32(0x0204 + PHYD_BASE_ADDR, rddata);
	rddata = mmio_rd32(0x0224 + PHYD_BASE_ADDR);
	rddata = modified_bits_by_value(rddata, 1, 18, 18);
	mmio_wr32(0x0224 + PHYD_BASE_ADDR, rddata);
	uartlog("[KC Info] --> en clock gated for power save !!!\n");

	// power on 3
}

void cvx16_wait_for_dfi_init_complete(void)
{
	uartlog("%s\n", __func__);
	ddr_debug_wr32(0x13);
	ddr_debug_num_write();
	// synp setting
	while (1) {
		rddata = mmio_rd32(cfg_base + 0x000001bc);
		//} while ((rddata & 0x00000001) != 1);
		if (get_bits_from_value(rddata, 0, 0) == 1) {
			break;
		}
	}
	mmio_wr32(cfg_base + 0x00000320, 0x00000000);
	rddata = mmio_rd32(cfg_base + 0x000001b0);
	rddata = modified_bits_by_value(rddata, 5, 5, 0);
	mmio_wr32(cfg_base + 0x000001b0, rddata);
	mmio_wr32(cfg_base + 0x00000320, 0x00000001);
	KC_MSG("dfi_init_complete finish\n");
}

void cvx16_ctrlupd_short(void)
{
	// ctrlupd short
	uartlog("cvx16ctrlupd_short\n");
	ddr_debug_wr32(0x2a);
	ddr_debug_num_write();
	// for gate track
	// dis_auto_ctrlupd [31] =0, dis_auto_ctrlupd_srx [30] =0, ctrlupd_pre_srx [29] =1 @ 0x1a0
	rddata = mmio_rd32(cfg_base + 0x000001a0);
	// original        rddata[31:29] = 0b001;
	rddata = modified_bits_by_value(rddata, 1, 31, 29);
	mmio_wr32(cfg_base + 0x000001a0, rddata);
	// dfi_t_ctrlupd_interval_min_x1024 [23:16] = 4, dfi_t_ctrlupd_interval_max_x1024 [7:0] = 8 @ 0x1a4
	rddata = mmio_rd32(cfg_base + 0x000001a4);
	// original        rddata[23:16] = 0x04;
	rddata = modified_bits_by_value(rddata, 4, 23, 16);
	// original        rddata[7:0] = 0x08;
	rddata = modified_bits_by_value(rddata, 8, 7, 0);
	mmio_wr32(cfg_base + 0x000001a4, rddata);
}

void cvx16_polling_dfi_init_start(void)
{
	uartlog("%s\n", __func__);
	ddr_debug_wr32(0x11);
	ddr_debug_num_write();
	while (1) {
		rddata = mmio_rd32(0x3028 + PHYD_BASE_ADDR);
		if ((get_bits_from_value(rddata, 8, 8) == 1)) {
			break;
		}
	}
}

void cvx16_set_dfi_init_complete(void)
{
	uartlog("%s\n", __func__);
	ddr_debug_wr32(0x48);
	ddr_debug_num_write();
#ifdef REAL_LOCK
	opdelay(20000);
#endif
	// rddata[8] = 1;
	rddata = 0x00000010;
	mmio_wr32(0x0120 + PHYD_BASE_ADDR, rddata);
	KC_MSG("set init_complete = 1 ...\n");

	// param_phyd_clkctrl_init_complete   <= int_regin[0];
	rddata = 0x00000001;
	mmio_wr32(0x0118 + PHYD_BASE_ADDR, rddata);
}

void cvx16_polling_synp_normal_mode(void)
{
	uartlog("%s\n", __func__);
	ddr_debug_wr32(0x14);
	ddr_debug_num_write();
	// synp ctrl operating_mode
	while (1) {
		rddata = mmio_rd32(cfg_base + 0x00000004);
		KC_MSG("operating_mode = %x\n", get_bits_from_value(rddata, 2, 0));

		if (get_bits_from_value(rddata, 2, 0) == 1) {
			break;
		}
	}
}

void cvx16_dfi_ca_park_prbs(uint32_t cap_enable)
{
	uint32_t dfi_ca_park_misc;
	uint32_t dfi_ca_park_retain_cycle;
	uint32_t dfi_ca_park_ca_ref;
	uint32_t dfi_ca_park_ca_park;

	// param_phyd_sw_dfi_phyupd_req =1
	rddata = 0x00000001;
	mmio_wr32(0x0174 + PHYD_BASE_ADDR, rddata);
	while (1) {
		// param_phyd_to_reg_dfi_phyupd_req  8   8
		// param_phyd_to_reg_dfi_phyupd_ack  9   9
		rddata = mmio_rd32(0x3030 + PHYD_BASE_ADDR);
		if (get_bits_from_value(rddata, 9, 8) == 3) {
			break;
		}
	}
	// DDR3
	//   cfg_det_en = 0b1;
	//   cfg_cs_det_en = 0b1;
	//   cap_prbs_en = 0b1;
	//   cfg_cs_polarity = 0b1;
	//   cap_prbs_1t = 0b0;
	//   cfg_ca_reference = {0b0,0x0_ffff,0x7,0x0,0b1,0b0,0b1,0b1};
	//   cfg_cs_retain_cycle = 0b0000_0001;
	//   cfg_ca_retain_cycle = 0b0000_0000;
	//   cfg_ca_park_value = 0x3fff_ffff;
	if (cap_enable == 1) {
		dfi_ca_park_misc = 0x1B;
		mmio_wr32(DDR_TOP_BASE + 0x00, dfi_ca_park_misc);
		KC_MSG("dfi_ca_park_prbs enable = 1\n");
	} else {
		dfi_ca_park_misc = 0x0;
		mmio_wr32(DDR_TOP_BASE + 0x00, dfi_ca_park_misc);
		KC_MSG("dfi_ca_park_prbs enable = 0\n");
	}
	dfi_ca_park_retain_cycle = 0x01;
	mmio_wr32(DDR_TOP_BASE + 0x04, dfi_ca_park_retain_cycle);
	dfi_ca_park_ca_ref = 0x1ffffcb;
	mmio_wr32(DDR_TOP_BASE + 0x08, dfi_ca_park_ca_ref);
	dfi_ca_park_ca_park = 0x3fffffff;
	mmio_wr32(DDR_TOP_BASE + 0x0c, dfi_ca_park_ca_park);

	// param_phyd_sw_dfi_phyupd_req_clr =1
	rddata = 0x00000010;
	mmio_wr32(0x0174 + PHYD_BASE_ADDR, rddata);
}


void cvx16_rdglvl_req(void)
{
	uint32_t selfref_sw;
	uint32_t en_dfi_dram_clk_disable;
	uint32_t powerdown_en;
	uint32_t selfref_en;
#ifdef DDR3
	uint32_t ddr3_mpr_mode;
#endif //DDR3
#ifdef DDR2_3
		uint32_t ddr3_mpr_mode;
#endif
	uint32_t port_num;
	// Note: training need ctrl_low_patch first
	//  Write 0 to PCTRL_n.port_en, without port 0
	//  port number = 0,1,2,3
	port_num = 0x4;
	for (int i = 1; i < port_num; i++) {
		mmio_wr32(cfg_base + 0x490 + 0xb0 * i, 0x0);
	}
	// Poll PSTAT.rd_port_busy_n = 0
	// Poll PSTAT.wr_port_busy_n = 0
	while (1) {
		rddata = mmio_rd32(cfg_base + 0x3fc);
		KC_MSG("Poll PSTAT.rd_port_busy_n =0\n");

		if (rddata == 0) {
			break;
		}
	}
	// disable PWRCTL.powerdown_en, PWRCTL.selfref_en
	rddata = mmio_rd32(cfg_base + 0x30);
	selfref_sw = get_bits_from_value(rddata, 5, 5);
	en_dfi_dram_clk_disable = get_bits_from_value(rddata, 3, 3);
	powerdown_en = get_bits_from_value(rddata, 1, 1);
	selfref_en = get_bits_from_value(rddata, 0, 0);
	rddata = modified_bits_by_value(rddata, 0, 5, 5); // PWRCTL.selfref_sw
	rddata = modified_bits_by_value(rddata, 0, 3, 3); // PWRCTL.en_dfi_dram_clk_disable
	// rddata=modified_bits_by_value(rddata, 0, 2, 2); //PWRCTL.deeppowerdown_en, non-mDDR/non-LPDDR2/non-LPDDR3,
							   // this register must not be set to 1
	rddata = modified_bits_by_value(rddata, 0, 1, 1); // PWRCTL.powerdown_en
	rddata = modified_bits_by_value(rddata, 0, 0, 0); // PWRCTL.selfref_en
	mmio_wr32(cfg_base + 0x30, rddata);
	cvx16_clk_gating_disable();
	// RFSHCTL3.dis_auto_refresh =1
	// rddata = mmio_rd32(cfg_base + 0x60);
	// rddata=modified_bits_by_value(rddata, 1, 0, 0); //RFSHCTL3.dis_auto_refresh
	// mmio_wr32(cfg_base + 0x60, rddata);
	uartlog("%s\n", __func__);
	ddr_debug_wr32(0x2f);
	ddr_debug_num_write();
	KC_MSG_TR("%s\n", __func__);

#ifdef DDR3
	rddata = mmio_rd32(0x0184 + PHYD_BASE_ADDR);
	ddr3_mpr_mode = get_bits_from_value(rddata, 4, 4);
	if (ddr3_mpr_mode) {
		// RFSHCTL3.dis_auto_refresh =1
		rddata = mmio_rd32(cfg_base + 0x60);
		rddata = modified_bits_by_value(rddata, 1, 0, 0); // RFSHCTL3.dis_auto_refresh
		mmio_wr32(cfg_base + 0x60, rddata);
		// MR3
		rddata = mmio_rd32(cfg_base + 0xe0);
		rddata = modified_bits_by_value(rddata, 1, 2, 2); // Dataflow from MPR
		cvx16_synp_mrw(0x3, get_bits_from_value(rddata, 15, 0));
	}
#endif
#ifdef DDR2_3

	rddata = mmio_rd32(0x0184 + PHYD_BASE_ADDR);
	ddr3_mpr_mode = get_bits_from_value(rddata, 4, 4);
	if (get_ddr_type() == DDR_TYPE_DDR3) {
		if (ddr3_mpr_mode) {
			// RFSHCTL3.dis_auto_refresh =1
			rddata = mmio_rd32(cfg_base + 0x60);
			rddata = modified_bits_by_value(rddata, 1, 0, 0); // RFSHCTL3.dis_auto_refresh
			mmio_wr32(cfg_base + 0x60, rddata);
			// MR3
			rddata = mmio_rd32(cfg_base + 0xe0);
			rddata = modified_bits_by_value(rddata, 1, 2, 2); // Dataflow from MPR
			cvx16_synp_mrw(0x3, get_bits_from_value(rddata, 15, 0));
		}
	}
#endif
	// bist setting for dfi rdglvl
	cvx16_bist_rdglvl_init();
	rddata = mmio_rd32(0x0184 + PHYD_BASE_ADDR);
	rddata = modified_bits_by_value(rddata, 1, 0, 0); // param_phyd_dfi_rdglvl_req
	mmio_wr32(0x0184 + PHYD_BASE_ADDR, rddata);
	KC_MSG_TR("wait retraining finish ...\n");

	while (1) {
		//[0] param_phyd_dfi_wrlvl_done
		//[1] param_phyd_dfi_rdglvl_done
		//[2] param_phyd_dfi_rdlvl_done
		//[3] param_phyd_dfi_wdqlvl_done
		rddata = mmio_rd32(0x3444 + PHYD_BASE_ADDR);
		if (get_bits_from_value(rddata, 1, 1) == 0x1) {
			// bist clock disable
			mmio_wr32(DDR_BIST_BASE + 0x0, 0x00040000);
			break;
		}
	}
#ifdef DDR3
	if (ddr3_mpr_mode) {
		// MR3
		rddata = mmio_rd32(cfg_base + 0xe0);
		rddata = modified_bits_by_value(rddata, 0, 2, 2); // Normal operation
		cvx16_synp_mrw(0x3, get_bits_from_value(rddata, 15, 0));
		// RFSHCTL3.dis_auto_refresh =0
		rddata = mmio_rd32(cfg_base + 0x60);
		rddata = modified_bits_by_value(rddata, 0, 0, 0); // RFSHCTL3.dis_auto_refresh
		mmio_wr32(cfg_base + 0x60, rddata);
	}
	uartlog("%s\n", __func__);
	ddr_debug_wr32(0x2f);
	ddr_debug_num_write();
#endif
#ifdef DDR2_3
	if (get_ddr_type() == DDR_TYPE_DDR3) {
		if (ddr3_mpr_mode) {
			// MR3
			rddata = mmio_rd32(cfg_base + 0xe0);
			rddata = modified_bits_by_value(rddata, 0, 2, 2); // Normal operation
			cvx16_synp_mrw(0x3, get_bits_from_value(rddata, 15, 0));
			// RFSHCTL3.dis_auto_refresh =0
			rddata = mmio_rd32(cfg_base + 0x60);
			rddata = modified_bits_by_value(rddata, 0, 0, 0); // RFSHCTL3.dis_auto_refresh
			mmio_wr32(cfg_base + 0x60, rddata);
		}
		uartlog("%s\n", __func__);
		ddr_debug_wr32(0x2f);
		ddr_debug_num_write();
	}
#endif
	// RFSHCTL3.dis_auto_refresh =0
	// rddata = mmio_rd32(cfg_base + 0x60);
	// rddata=modified_bits_by_value(rddata, 0, 0, 0); //RFSHCTL3.dis_auto_refresh
	// mmio_wr32(cfg_base + 0x60, rddata);
	// restore PWRCTL.powerdown_en, PWRCTL.selfref_en
	rddata = mmio_rd32(cfg_base + 0x30);
	rddata = modified_bits_by_value(rddata, selfref_sw, 5, 5); // PWRCTL.selfref_sw
	rddata = modified_bits_by_value(rddata, en_dfi_dram_clk_disable, 3, 3); // PWRCTL.en_dfi_dram_clk_disable
	// rddata=modified_bits_by_value(rddata, 0, 2, 2); //PWRCTL.deeppowerdown_en, non-mDDR/non-LPDDR2/non-LPDDR3,
							   //this register must not be set to 1
	rddata = modified_bits_by_value(rddata, powerdown_en, 1, 1); // PWRCTL.powerdown_en
	rddata = modified_bits_by_value(rddata, selfref_en, 0, 0); // PWRCTL.selfref_en
	mmio_wr32(cfg_base + 0x30, rddata);
	// Write 1 to PCTRL_n.port_en
	for (int i = 1; i < port_num; i++) {
		mmio_wr32(cfg_base + 0x490 + 0xb0 * i, 0x1);
	}
	// cvx16_rdglvl_status();
	cvx16_clk_gating_enable();
}

void cvx16_rdlvl_req(uint32_t mode)
{
	uint32_t selfref_sw;
	uint32_t en_dfi_dram_clk_disable;
	uint32_t powerdown_en;
	uint32_t selfref_en;
#ifdef DDR3
	uint32_t ddr3_mpr_mode;
#endif //DDR3
#ifdef DDR2_3
	uint32_t ddr3_mpr_mode;
#endif
	uint32_t port_num;
	uint32_t vref_training_en;
	// uint32_t code_neg; //unused
	// uint32_t code_pos; //unused
	// Note: training need ctrl_low_patch first
	// mode = 0x0  : MPR mode, DDR3 only.
	// mode = 0x1  : sram write/read continuous goto
	// mode = 0x2  : multi- bist write/read
	// mode = 0x10 : with Error enject,  multi- bist write/read
	// mode = 0x12 : with Error enject,  multi- bist write/read
	//  Write 0 to PCTRL_n.port_en, without port 0
	//  port number = 0,1,2,3
	port_num = 0x4;
	for (int i = 1; i < port_num; i++) {
		mmio_wr32(cfg_base + 0x490 + 0xb0 * i, 0x0);
	}
	// Poll PSTAT.rd_port_busy_n = 0
	// Poll PSTAT.wr_port_busy_n = 0
	while (1) {
		rddata = mmio_rd32(cfg_base + 0x3fc);
		KC_MSG("Poll PSTAT.rd_port_busy_n =0\n");

		if (rddata == 0) {
			break;
		}
	}
	// disable PWRCTL.powerdown_en, PWRCTL.selfref_en
	rddata = mmio_rd32(cfg_base + 0x30);
	selfref_sw = get_bits_from_value(rddata, 5, 5);
	en_dfi_dram_clk_disable = get_bits_from_value(rddata, 3, 3);
	powerdown_en = get_bits_from_value(rddata, 1, 1);
	selfref_en = get_bits_from_value(rddata, 0, 0);
	rddata = modified_bits_by_value(rddata, 0, 5, 5); // PWRCTL.selfref_sw
	rddata = modified_bits_by_value(rddata, 0, 3, 3); // PWRCTL.en_dfi_dram_clk_disable
	// rddata=modified_bits_by_value(rddata, 0, 2, 2); //PWRCTL.deeppowerdown_en, non-mDDR/non-LPDDR2/non-LPDDR3,
							   //this register must not be set to 1
	rddata = modified_bits_by_value(rddata, 0, 1, 1); // PWRCTL.powerdown_en
	rddata = modified_bits_by_value(rddata, 0, 0, 0); // PWRCTL.selfref_en
	mmio_wr32(cfg_base + 0x30, rddata);
	cvx16_clk_gating_disable();
	//    //RFSHCTL3.dis_auto_refresh =1
	//    rddata = mmio_rd32(cfg_base + 0x60);
	//    rddata=modified_bits_by_value(rddata, 1, 0, 0); //RFSHCTL3.dis_auto_refresh
	//    mmio_wr32(cfg_base + 0x60, rddata);
	uartlog("%s\n", __func__);
	ddr_debug_wr32(0x30);
	ddr_debug_num_write();
	cvx16_dfi_ca_park_prbs(1);
	KC_MSG("%s\n", __func__);

	//deskew start from 0x20
	rddata = mmio_rd32(0x0080 + PHYD_BASE_ADDR);
	rddata = modified_bits_by_value(rddata, 0x20, 22, 16); //param_phyd_pirdlvl_deskew_start
	rddata = modified_bits_by_value(rddata, 0x1F, 30, 24); //param_phyd_pirdlvl_deskew_end
	mmio_wr32(0x0080 + PHYD_BASE_ADDR,  rddata);

	// save param_phyd_pirdlvl_vref_training_en
	rddata = mmio_rd32(0x008c + PHYD_BASE_ADDR);
	vref_training_en = get_bits_from_value(rddata, 2, 2);
	rddata = modified_bits_by_value(rddata, 0, 1, 1); // param_phyd_pirdlvl_rx_init_deskew_en
	rddata = modified_bits_by_value(rddata, 0, 2, 2); // param_phyd_pirdlvl_vref_training_en
	rddata = modified_bits_by_value(rddata, 0, 3, 3); // param_phyd_pirdlvl_rdvld_training_en = 0
	mmio_wr32(0x008c + PHYD_BASE_ADDR, rddata);
#ifdef DDR3
	rddata = mmio_rd32(0x0188 + PHYD_BASE_ADDR);
	ddr3_mpr_mode = get_bits_from_value(rddata, 4, 4);
	if (ddr3_mpr_mode) {
		// RFSHCTL3.dis_auto_refresh =1
		rddata = mmio_rd32(cfg_base + 0x60);
		rddata = modified_bits_by_value(rddata, 1, 0, 0); // RFSHCTL3.dis_auto_refresh
		mmio_wr32(cfg_base + 0x60, rddata);
		// MR3
		rddata = mmio_rd32(cfg_base + 0xe0);
		rddata = modified_bits_by_value(rddata, 1, 2, 2); // Dataflow from MPR
		cvx16_synp_mrw(0x3, get_bits_from_value(rddata, 15, 0));
	}
#endif
#ifdef DDR2_3
	rddata = mmio_rd32(0x0188 + PHYD_BASE_ADDR);
	ddr3_mpr_mode = get_bits_from_value(rddata, 4, 4);
	if (get_ddr_type() == DDR_TYPE_DDR3) {
		if (ddr3_mpr_mode) {
			// RFSHCTL3.dis_auto_refresh =1
			rddata = mmio_rd32(cfg_base + 0x60);
			rddata = modified_bits_by_value(rddata, 1, 0, 0); // RFSHCTL3.dis_auto_refresh
			mmio_wr32(cfg_base + 0x60, rddata);
			// MR3
			rddata = mmio_rd32(cfg_base + 0xe0);
			rddata = modified_bits_by_value(rddata, 1, 2, 2); // Dataflow from MPR
			cvx16_synp_mrw(0x3, get_bits_from_value(rddata, 15, 0));
		}
	}
#endif
	// bist setting for dfi rdglvl
	cvx16_bist_rdlvl_init(mode);
	rddata = mmio_rd32(0x0188 + PHYD_BASE_ADDR);
	rddata = modified_bits_by_value(rddata, 1, 0, 0); // param_phyd_dfi_rdlvl_req
	mmio_wr32(0x0188 + PHYD_BASE_ADDR, rddata);
	KC_MSG("dfi_rdlvl_req 1\n");

	KC_MSG("wait retraining finish ...\n");

	while (1) {
		//[0] param_phyd_dfi_wrlvl_done
		//[1] param_phyd_dfi_rdglvl_done
		//[2] param_phyd_dfi_rdlvl_done
		//[3] param_phyd_dfi_wdqlvl_done
		rddata = mmio_rd32(0x3444 + PHYD_BASE_ADDR);
		if (get_bits_from_value(rddata, 2, 2) == 0x1) {
			break;
		}
	}
	if (vref_training_en == 0x1) {
		rddata = mmio_rd32(0x008c + PHYD_BASE_ADDR);
		rddata = modified_bits_by_value(rddata, 0, 2, 2); // param_phyd_pirdlvl_vref_training_en
		mmio_wr32(0x008c + PHYD_BASE_ADDR, rddata);
		// final training, keep rx trig_lvl
		KC_MSG("final training, keep rx trig_lvl\n");

		rddata = mmio_rd32(0x0188 + PHYD_BASE_ADDR);
		rddata = modified_bits_by_value(rddata, 1, 0, 0); // param_phyd_dfi_rdlvl_req
		mmio_wr32(0x0188 + PHYD_BASE_ADDR, rddata);
		KC_MSG("dfi_rdlvl_req 2\n");

		KC_MSG("wait retraining finish ...\n");

		while (1) {
			//[0] param_phyd_dfi_wrlvl_done
			//[1] param_phyd_dfi_rdglvl_done
			//[2] param_phyd_dfi_rdlvl_done
			//[3] param_phyd_dfi_wdqlvl_done
			rddata = mmio_rd32(0x3444 + PHYD_BASE_ADDR);
			if (get_bits_from_value(rddata, 2, 2) == 0x1) {
				break;
			}
		}
		rddata = mmio_rd32(0x008c + PHYD_BASE_ADDR);
		rddata = modified_bits_by_value(rddata, vref_training_en, 2, 2); // param_phyd_pirdlvl_vref_training_en
		mmio_wr32(0x008c + PHYD_BASE_ADDR, rddata);
	}

#ifdef DDR3
	if (ddr3_mpr_mode) {
		// MR3
		rddata = mmio_rd32(cfg_base + 0xe0);
		rddata = modified_bits_by_value(rddata, 0, 2, 2); // Normal operation
		cvx16_synp_mrw(0x3, get_bits_from_value(rddata, 15, 0));
		// RFSHCTL3.dis_auto_refresh =0
		rddata = mmio_rd32(cfg_base + 0x60);
		rddata = modified_bits_by_value(rddata, 0, 0, 0); // RFSHCTL3.dis_auto_refresh
		mmio_wr32(cfg_base + 0x60, rddata);
	}
	uartlog("%s\n", __func__);
	ddr_debug_wr32(0x30);
	ddr_debug_num_write();
#endif
#ifdef DDR2_3
	if (get_ddr_type() == DDR_TYPE_DDR3) {
		if (ddr3_mpr_mode) {
			// MR3
			rddata = mmio_rd32(cfg_base + 0xe0);
			rddata = modified_bits_by_value(rddata, 0, 2, 2); // Normal operation
			cvx16_synp_mrw(0x3, get_bits_from_value(rddata, 15, 0));
			// RFSHCTL3.dis_auto_refresh =0
			rddata = mmio_rd32(cfg_base + 0x60);
			rddata = modified_bits_by_value(rddata, 0, 0, 0); // RFSHCTL3.dis_auto_refresh
			mmio_wr32(cfg_base + 0x60, rddata);
		}
		uartlog("%s\n", __func__);
		ddr_debug_wr32(0x30);
		ddr_debug_num_write();
	}
#endif

	cvx16_rdvld_train();

	//    //RFSHCTL3.dis_auto_refresh =0
	//    rddata = mmio_rd32(cfg_base + 0x60);
	//    rddata=modified_bits_by_value(rddata, 0, 0, 0); //RFSHCTL3.dis_auto_refresh
	//    mmio_wr32(cfg_base + 0x60, rddata);
	// bist clock disable
	mmio_wr32(DDR_BIST_BASE + 0x0, 0x00040000);
	cvx16_dfi_ca_park_prbs(0);
	// restore PWRCTL.powerdown_en, PWRCTL.selfref_en
	rddata = mmio_rd32(cfg_base + 0x30);
	rddata = modified_bits_by_value(rddata, selfref_sw, 5, 5); // PWRCTL.selfref_sw
	rddata = modified_bits_by_value(rddata, en_dfi_dram_clk_disable, 3, 3); // PWRCTL.en_dfi_dram_clk_disable
	// rddata=modified_bits_by_value(rddata, 0, 2, 2); //PWRCTL.deeppowerdown_en, non-mDDR/non-LPDDR2/non-LPDDR3,
							   //this register must not be set to 1
	rddata = modified_bits_by_value(rddata, powerdown_en, 1, 1); // PWRCTL.powerdown_en
	rddata = modified_bits_by_value(rddata, selfref_en, 0, 0); // PWRCTL.selfref_en
	mmio_wr32(cfg_base + 0x30, rddata);
	// Write 1 to PCTRL_n.port_en
	for (int i = 1; i < port_num; i++) {
		mmio_wr32(cfg_base + 0x490 + 0xb0 * i, 0x1);
	}
	// cvx16_rdlvl_status();
	cvx16_clk_gating_enable();
}

void cvx16_rdlvl_sw_req(uint32_t mode)
{
	uint32_t selfref_sw;
	uint32_t en_dfi_dram_clk_disable;
	uint32_t powerdown_en;
	uint32_t selfref_en;
#ifdef DDR3
	uint32_t ddr3_mpr_mode;
#endif //DDR3
#ifdef DDR2_3
	uint32_t ddr3_mpr_mode;
#endif
	uint32_t port_num;
	uint32_t vref_training_en;
	uint32_t byte0_pirdlvl_sw_upd_ack;
	uint32_t byte1_pirdlvl_sw_upd_ack;
	uint32_t rx_vref_sel;
	uint32_t byte0_data_rise_fail;
	uint32_t byte0_data_fall_fail;
	uint32_t byte1_data_rise_fail;
	uint32_t byte1_data_fall_fail;
	uint32_t dlie_code;
	uint32_t byte0_all_le_found;
	uint32_t byte0_all_te_found;
	uint32_t byte1_all_le_found;
	uint32_t byte1_all_te_found;
	// uint32_t code_neg; //unused
	// uint32_t code_pos; //unused
	uint32_t byte0_cur_pirdlvl_st;
	uint32_t byte1_cur_pirdlvl_st;
	uint32_t sw_upd_req_start;

	sw_upd_req_start = 0;
	// mode = 0x0  : MPR mode, DDR3 only.
	// mode = 0x1  : sram write/read continuous goto
	// mode = 0x2  : multi- bist write/read
	// mode = 0x10 : with Error enject,  multi- bist write/read
	// mode = 0x12 : with Error enject,  multi- bist write/read
	//  Write 0 to PCTRL_n.port_en, without port 0
	//  port number = 0,1,2,3
	port_num = 0x4;
	for (int i = 1; i < port_num; i++) {
		mmio_wr32(cfg_base + 0x490 + 0xb0 * i, 0x0);
	}
	// Poll PSTAT.rd_port_busy_n = 0
	// Poll PSTAT.wr_port_busy_n = 0
	while (1) {
		rddata = mmio_rd32(cfg_base + 0x3fc);
		KC_MSG("Poll PSTAT.rd_port_busy_n =0\n");

		if (rddata == 0) {
			break;
		}
	}
	// disable PWRCTL.powerdown_en, PWRCTL.selfref_en
	rddata = mmio_rd32(cfg_base + 0x30);
	selfref_sw = get_bits_from_value(rddata, 5, 5);
	en_dfi_dram_clk_disable = get_bits_from_value(rddata, 3, 3);
	powerdown_en = get_bits_from_value(rddata, 1, 1);
	selfref_en = get_bits_from_value(rddata, 0, 0);
	rddata = modified_bits_by_value(rddata, 0, 5, 5); // PWRCTL.selfref_sw
	rddata = modified_bits_by_value(rddata, 0, 3, 3); // PWRCTL.en_dfi_dram_clk_disable
	// rddata=modified_bits_by_value(rddata, 0, 2, 2); //PWRCTL.deeppowerdown_en, non-mDDR/non-LPDDR2/non-LPDDR3,
							   //this register must not be set to 1
	rddata = modified_bits_by_value(rddata, 0, 1, 1); // PWRCTL.powerdown_en
	rddata = modified_bits_by_value(rddata, 0, 0, 0); // PWRCTL.selfref_en
	mmio_wr32(cfg_base + 0x30, rddata);
	cvx16_clk_gating_disable();
	//    //RFSHCTL3.dis_auto_refresh =1
	//    rddata = mmio_rd32(cfg_base + 0x60);
	//    rddata=modified_bits_by_value(rddata, 1, 0, 0); //RFSHCTL3.dis_auto_refresh
	//    mmio_wr32(cfg_base + 0x60, rddata);
	uartlog("%s\n", __func__);
	ddr_debug_wr32(0x30);
	ddr_debug_num_write();
	cvx16_dfi_ca_park_prbs(1);
	KC_MSG("%s\n", __func__);

	//deskew start from 0x20
	rddata = mmio_rd32(0x0080 + PHYD_BASE_ADDR);
	rddata = modified_bits_by_value(rddata, 0x20, 22, 16); //param_phyd_pirdlvl_deskew_start
	rddata = modified_bits_by_value(rddata, 0x1F, 30, 24); //param_phyd_pirdlvl_deskew_end
	mmio_wr32(0x0080 + PHYD_BASE_ADDR,  rddata);

	// save param_phyd_pirdlvl_vref_training_en
	rddata = mmio_rd32(0x008c + PHYD_BASE_ADDR);
	vref_training_en = get_bits_from_value(rddata, 2, 2);
	rddata = modified_bits_by_value(rddata, 0, 1, 1); // param_phyd_pirdlvl_rx_init_deskew_en
	rddata = modified_bits_by_value(rddata, 0, 2, 2); // param_phyd_pirdlvl_vref_training_en
	rddata = modified_bits_by_value(rddata, 0, 3, 3); // param_phyd_pirdlvl_rdvld_training_en = 0
	mmio_wr32(0x008c + PHYD_BASE_ADDR, rddata);

#ifdef DDR3
	rddata = mmio_rd32(0x0188 + PHYD_BASE_ADDR);
	ddr3_mpr_mode = get_bits_from_value(rddata, 4, 4);
	if (ddr3_mpr_mode) {
		// RFSHCTL3.dis_auto_refresh =1
		rddata = mmio_rd32(cfg_base + 0x60);
		rddata = modified_bits_by_value(rddata, 1, 0, 0); // RFSHCTL3.dis_auto_refresh
		mmio_wr32(cfg_base + 0x60, rddata);
		// MR3
		rddata = mmio_rd32(cfg_base + 0xe0);
		rddata = modified_bits_by_value(rddata, 1, 2, 2); // Dataflow from MPR
		cvx16_synp_mrw(0x3, get_bits_from_value(rddata, 15, 0));
	}
#endif
#ifdef DDR2_3

	rddata = mmio_rd32(0x0188 + PHYD_BASE_ADDR);
	ddr3_mpr_mode = get_bits_from_value(rddata, 4, 4);
	if (get_ddr_type() == DDR_TYPE_DDR3) {
		if (ddr3_mpr_mode) {
			// RFSHCTL3.dis_auto_refresh =1
			rddata = mmio_rd32(cfg_base + 0x60);
			rddata = modified_bits_by_value(rddata, 1, 0, 0); // RFSHCTL3.dis_auto_refresh
			mmio_wr32(cfg_base + 0x60, rddata);
			// MR3
			rddata = mmio_rd32(cfg_base + 0xe0);
			rddata = modified_bits_by_value(rddata, 1, 2, 2); // Dataflow from MPR
			cvx16_synp_mrw(0x3, get_bits_from_value(rddata, 15, 0));
		}
	}
#endif

	// bist setting for dfi rdglvl
	cvx16_bist_rdlvl_init(mode);
	// SW mode
	rddata = mmio_rd32(0x0090 + PHYD_BASE_ADDR);
	rddata = modified_bits_by_value(rddata, 1, 9, 9); // param_phyd_pirdlvl_sw
	mmio_wr32(0x0090 + PHYD_BASE_ADDR, rddata);
	rddata = mmio_rd32(0x0188 + PHYD_BASE_ADDR);
	rddata = modified_bits_by_value(rddata, 1, 0, 0); // param_phyd_dfi_rdlvl_req
	mmio_wr32(0x0188 + PHYD_BASE_ADDR, rddata);
	KC_MSG("dfi_rdlvl_req 1\n");

	KC_MSG("wait retraining finish ...\n");

	while (1) {
		//[0] param_phyd_dfi_wrlvl_done
		//[1] param_phyd_dfi_rdglvl_done
		//[2] param_phyd_dfi_rdlvl_done
		//[3] param_phyd_dfi_wdqlvl_done
		rddata = mmio_rd32(0x3444 + PHYD_BASE_ADDR);
		if (get_bits_from_value(rddata, 2, 2) == 0x1) {
			break;
		}
		while (1) {
			rddata = mmio_rd32(0x31B0 + PHYD_BASE_ADDR);
			byte0_pirdlvl_sw_upd_ack = get_bits_from_value(rddata, 0, 0);
			byte0_all_le_found = get_bits_from_value(rddata, 5, 5) & get_bits_from_value(rddata, 7, 7);
			byte0_all_te_found = get_bits_from_value(rddata, 4, 4) & get_bits_from_value(rddata, 6, 6);
			byte0_cur_pirdlvl_st = get_bits_from_value(rddata, 23, 20);
			rddata = mmio_rd32(0x31B0 + 0x40 + PHYD_BASE_ADDR);
			byte1_pirdlvl_sw_upd_ack = get_bits_from_value(rddata, 0, 0);
			byte1_all_le_found = get_bits_from_value(rddata, 5, 5) & get_bits_from_value(rddata, 7, 7);
			byte1_all_te_found = get_bits_from_value(rddata, 4, 4) & get_bits_from_value(rddata, 6, 6);
			byte1_cur_pirdlvl_st = get_bits_from_value(rddata, 23, 20);
			KC_MSG("=1 byte0_pirdlvl_sw_upd_ack = %x, byte1_pirdlvl_sw_upd_ack = %x\n",
			       byte0_pirdlvl_sw_upd_ack, byte1_pirdlvl_sw_upd_ack);

			if ((byte0_all_le_found == 0) && (byte0_all_te_found == 0) && (byte1_all_le_found == 0) &&
			    (byte1_all_te_found == 0)) {
				sw_upd_req_start = 0x1;
			} else {
				if ((byte0_all_le_found == 0x1) && (byte0_all_te_found == 0x1) &&
				    (byte1_all_le_found == 0x1) && (byte1_all_te_found == 0x1) &&
				    ((byte0_cur_pirdlvl_st == 0x0) && (byte1_cur_pirdlvl_st == 0x0))) {
					sw_upd_req_start = 0x0;
				}
			}
			// KC_MSG("sw_upd_req_start = %x\n", sw_upd_req_start);

			if (((byte0_pirdlvl_sw_upd_ack == 0x1) || (byte0_all_le_found & byte0_all_te_found)) &&
			    ((byte1_pirdlvl_sw_upd_ack == 0x1) || (byte1_all_le_found & byte1_all_te_found))) {
				rddata = mmio_rd32(0x0B24 + PHYD_BASE_ADDR);
				rx_vref_sel = get_bits_from_value(rddata, 4, 0);
				rddata = mmio_rd32(0x0B08 + PHYD_BASE_ADDR);
				dlie_code = get_bits_from_value(rddata, 15, 8);
				rddata = mmio_rd32(0x31B4 + PHYD_BASE_ADDR);
				if (byte0_all_te_found) {
					byte0_data_rise_fail = 0xff;
					byte0_data_fall_fail = 0xff;
				} else {
					byte0_data_rise_fail = get_bits_from_value(rddata, 24, 16);
					byte0_data_fall_fail = get_bits_from_value(rddata, 8, 0);
				}
				rddata = mmio_rd32(0x31B4 + 0x40 + PHYD_BASE_ADDR);
				if (byte1_all_te_found) {
					byte1_data_rise_fail = 0xff;
					byte1_data_fall_fail = 0xff;
				} else {
					byte1_data_rise_fail = get_bits_from_value(rddata, 24, 16);
					byte1_data_fall_fail = get_bits_from_value(rddata, 8, 0);
				}
				if (((byte0_pirdlvl_sw_upd_ack == 0x1) || (byte1_pirdlvl_sw_upd_ack == 0x1)) &&
				    ((byte0_cur_pirdlvl_st != 0x0) && (byte1_cur_pirdlvl_st != 0x0))) {
					SHMOO_MSG("vref = %02x, sw_rdq_training_start = %08x , ",
						  rx_vref_sel, dlie_code);
					SHMOO_MSG("err_data_rise/err_data_fall = %08x, %08x\n",
						  ((byte0_data_rise_fail & 0x000000FF) |
						   ((byte1_data_rise_fail & 0x000000FF) << 8)),
						   ((byte0_data_fall_fail & 0x000000FF) |
						   ((byte1_data_fall_fail & 0x000000FF) << 8)));
				}
				//if (((byte0_pirdlvl_sw_upd_ack == 0x1) || (byte1_pirdlvl_sw_upd_ack == 0x1))
				//    || ((byte0_cur_pirdlvl_st != 0x0) && (byte1_cur_pirdlvl_st != 0x0))) {
				if (((byte0_pirdlvl_sw_upd_ack == 0x1) || (byte1_pirdlvl_sw_upd_ack == 0x1)) &&
				    (sw_upd_req_start == 0x1)) {
					rddata = mmio_rd32(0x0090 + PHYD_BASE_ADDR);
					rddata = modified_bits_by_value(rddata, 1, 10,
									10); // param_phyd_pirdlvl_sw_upd_req
					mmio_wr32(0x0090 + PHYD_BASE_ADDR, rddata);
				}
				KC_MSG("byte0_all_le_found, byte0_all_te_found, ");
				KC_MSG("byte1_all_le_found, byte1_all_te_found");
				KC_MSG(" =%x %x %x %x\n",
				       byte0_all_le_found, byte0_all_te_found,
				       byte1_all_le_found, byte1_all_te_found);

				break;
			}
			if ((byte0_all_le_found == 0x1) && (byte0_all_te_found == 0x1) &&
			    (byte1_all_le_found == 0x1) && (byte1_all_te_found == 0x1)) {
				break;
			}

			rddata = mmio_rd32(0x3444 + PHYD_BASE_ADDR);
			if (get_bits_from_value(rddata, 2, 2) == 0x1) {
				break;
			}
		}
	}
	if (vref_training_en == 0x1) {
		rddata = mmio_rd32(0x008c + PHYD_BASE_ADDR);
		rddata = modified_bits_by_value(rddata, 0, 2, 2); // param_phyd_pirdlvl_vref_training_en
		mmio_wr32(0x008c + PHYD_BASE_ADDR, rddata);
		// final training, keep rx trig_lvl
		KC_MSG("final training, keep rx trig_lvl\n");

		rddata = mmio_rd32(0x0188 + PHYD_BASE_ADDR);
		rddata = modified_bits_by_value(rddata, 1, 0, 0); // param_phyd_dfi_rdlvl_req
		mmio_wr32(0x0188 + PHYD_BASE_ADDR, rddata);
		KC_MSG("dfi_rdlvl_req 2\n");

		KC_MSG("wait retraining finish ...\n");

		while (1) {
			rddata = mmio_rd32(0x31B0 + PHYD_BASE_ADDR);
			byte0_pirdlvl_sw_upd_ack = get_bits_from_value(rddata, 0, 0);
			byte0_all_le_found = get_bits_from_value(rddata, 5, 5) & get_bits_from_value(rddata, 7, 7);
			byte0_all_te_found = get_bits_from_value(rddata, 4, 4) & get_bits_from_value(rddata, 6, 6);
			byte0_cur_pirdlvl_st = get_bits_from_value(rddata, 23, 20);
			rddata = mmio_rd32(0x31B0 + 0x40 + PHYD_BASE_ADDR);
			byte1_pirdlvl_sw_upd_ack = get_bits_from_value(rddata, 0, 0);
			byte1_all_le_found = get_bits_from_value(rddata, 5, 5) & get_bits_from_value(rddata, 7, 7);
			byte1_all_te_found = get_bits_from_value(rddata, 4, 4) & get_bits_from_value(rddata, 6, 6);
			byte1_cur_pirdlvl_st = get_bits_from_value(rddata, 23, 20);
			KC_MSG("Wait all_found clear  ");
			KC_MSG("byte0_all_le_found | byte0_all_te_found = %x, ",
			       (byte0_all_le_found | byte0_all_te_found));
			KC_MSG("byte1_all_le_found | byte1_all_te_found = %x\n",
			       (byte1_all_le_found | byte1_all_te_found));

			if ((byte0_all_le_found == 0) && (byte0_all_te_found == 0) &&
			    (byte1_all_le_found == 0) && (byte1_all_te_found == 0)) {
				break;
			}
		}
		while (1) {
			//[0] param_phyd_dfi_wrlvl_done
			//[1] param_phyd_dfi_rdglvl_done
			//[2] param_phyd_dfi_rdlvl_done
			//[3] param_phyd_dfi_wdqlvl_done
			rddata = mmio_rd32(0x3444 + PHYD_BASE_ADDR);
			if (get_bits_from_value(rddata, 2, 2) == 0x1) {
				break;
			}
			while (1) {
				rddata = mmio_rd32(0x31B0 + PHYD_BASE_ADDR);
				byte0_pirdlvl_sw_upd_ack = get_bits_from_value(rddata, 0, 0);
				byte0_all_le_found =
					get_bits_from_value(rddata, 5, 5) & get_bits_from_value(rddata, 7, 7);
				byte0_all_te_found =
					get_bits_from_value(rddata, 4, 4) & get_bits_from_value(rddata, 6, 6);
				byte0_cur_pirdlvl_st = get_bits_from_value(rddata, 23, 20);
				rddata = mmio_rd32(0x31B0 + 0x40 + PHYD_BASE_ADDR);
				byte1_pirdlvl_sw_upd_ack = get_bits_from_value(rddata, 0, 0);
				byte1_all_le_found =
					get_bits_from_value(rddata, 5, 5) & get_bits_from_value(rddata, 7, 7);
				byte1_all_te_found =
					get_bits_from_value(rddata, 4, 4) & get_bits_from_value(rddata, 6, 6);
				byte1_cur_pirdlvl_st = get_bits_from_value(rddata, 23, 20);
				KC_MSG("=1 byte0_pirdlvl_sw_upd_ack = %x, byte1_pirdlvl_sw_upd_ack = %x\n",
				       byte0_pirdlvl_sw_upd_ack, byte1_pirdlvl_sw_upd_ack);

				if ((byte0_all_le_found == 0) && (byte0_all_te_found == 0) &&
				    (byte1_all_le_found == 0) && (byte1_all_te_found == 0)) {
					sw_upd_req_start = 0x1;
				} else {
					if ((byte0_all_le_found == 0x1) && (byte0_all_te_found == 0x1) &&
					    (byte1_all_le_found == 0x1) && (byte1_all_te_found == 0x1) &&
					    ((byte0_cur_pirdlvl_st == 0x0) && (byte1_cur_pirdlvl_st == 0x0))) {
						sw_upd_req_start = 0x0;
					}
				}
				// KC_MSG("sw_upd_req_start = %x\n", sw_upd_req_start);

				if (((byte0_pirdlvl_sw_upd_ack == 0x1) || (byte0_all_le_found & byte0_all_te_found)) &&
				    ((byte1_pirdlvl_sw_upd_ack == 0x1) || (byte1_all_le_found & byte1_all_te_found))) {
					//if ((byte0_pirdlvl_sw_upd_ack == 0x1) && (byte1_pirdlvl_sw_upd_ack == 0x1)) {
					rddata = mmio_rd32(0x0B24 + PHYD_BASE_ADDR);
					rx_vref_sel = get_bits_from_value(rddata, 4, 0);
					rddata = mmio_rd32(0x0B08 + PHYD_BASE_ADDR);
					dlie_code = get_bits_from_value(rddata, 15, 8);
					rddata = mmio_rd32(0x31B4 + PHYD_BASE_ADDR);
					if (byte0_all_te_found) {
						byte0_data_rise_fail = 0xff;
						byte0_data_fall_fail = 0xff;
					} else {
						byte0_data_rise_fail = get_bits_from_value(rddata, 24, 16);
						byte0_data_fall_fail = get_bits_from_value(rddata, 8, 0);
					}
					rddata = mmio_rd32(0x31B4 + 0x40 + PHYD_BASE_ADDR);
					if (byte1_all_te_found) {
						byte1_data_rise_fail = 0xff;
						byte1_data_fall_fail = 0xff;
					} else {
						byte1_data_rise_fail = get_bits_from_value(rddata, 24, 16);
						byte1_data_fall_fail = get_bits_from_value(rddata, 8, 0);
					}
					if (((byte0_pirdlvl_sw_upd_ack == 0x1) || (byte1_pirdlvl_sw_upd_ack == 0x1)) &&
					    ((byte0_cur_pirdlvl_st != 0x0) && (byte1_cur_pirdlvl_st != 0x0))) {
						SHMOO_MSG("vref = %02x, sw_rdq_training_start = %08x , ",
							  rx_vref_sel, dlie_code);
						SHMOO_MSG("err_data_rise/err_data_fall = %08x, %08x\n",
						       ((byte0_data_rise_fail & 0x000000FF) |
							((byte1_data_rise_fail & 0x000000FF) << 8)),
						       ((byte0_data_fall_fail & 0x000000FF) |
							((byte1_data_fall_fail & 0x000000FF) << 8)));
					}
					// if (((byte0_pirdlvl_sw_upd_ack == 0x1) || (byte1_pirdlvl_sw_upd_ack == 0x1))
					//     || ((byte0_cur_pirdlvl_st != 0x0) && (byte1_cur_pirdlvl_st != 0x0))) {
					if (((byte0_pirdlvl_sw_upd_ack == 0x1) || (byte1_pirdlvl_sw_upd_ack == 0x1)) &&
					    (sw_upd_req_start == 0x1)) {
						rddata = mmio_rd32(0x0090 + PHYD_BASE_ADDR);
						rddata = modified_bits_by_value(rddata, 1, 10,
										10); // param_phyd_pirdlvl_sw_upd_req
						mmio_wr32(0x0090 + PHYD_BASE_ADDR, rddata);
					}
					KC_MSG("byte0_all_le_found, byte0_all_te_found, ");
					KC_MSG("byte1_all_le_found, byte1_all_te_found");
					KC_MSG(" =%x %x %x %x\n",
					       byte0_all_le_found, byte0_all_te_found,
					       byte1_all_le_found, byte1_all_te_found);

					break;
				}
				if ((byte0_all_le_found == 0x1) && (byte0_all_te_found == 0x1) &&
				    (byte1_all_le_found == 0x1) && (byte1_all_te_found == 0x1)) {
					break;
				}

				rddata = mmio_rd32(0x3444 + PHYD_BASE_ADDR);
				if (get_bits_from_value(rddata, 2, 2) == 0x1) {
					break;
				}
			}
		}
		rddata = mmio_rd32(0x008c + PHYD_BASE_ADDR);
		rddata = modified_bits_by_value(rddata, vref_training_en, 2, 2); // param_phyd_pirdlvl_vref_training_en
		mmio_wr32(0x008c + PHYD_BASE_ADDR, rddata);
	}

#ifdef DDR3
	if (ddr3_mpr_mode) {
		// MR3
		rddata = mmio_rd32(cfg_base + 0xe0);
		rddata = modified_bits_by_value(rddata, 0, 2, 2); // Normal operation
		cvx16_synp_mrw(0x3, get_bits_from_value(rddata, 15, 0));
		// RFSHCTL3.dis_auto_refresh =0
		rddata = mmio_rd32(cfg_base + 0x60);
		rddata = modified_bits_by_value(rddata, 0, 0, 0); // RFSHCTL3.dis_auto_refresh
		mmio_wr32(cfg_base + 0x60, rddata);
	}
	uartlog("cvx16_rdlvl_req\n");
	ddr_debug_wr32(0x30);
	ddr_debug_num_write();
#endif
#ifdef DDR2_3
	if (get_ddr_type() == DDR_TYPE_DDR3) {
		if (ddr3_mpr_mode) {
			// MR3
			rddata = mmio_rd32(cfg_base + 0xe0);
			rddata = modified_bits_by_value(rddata, 0, 2, 2); // Normal operation
			cvx16_synp_mrw(0x3, get_bits_from_value(rddata, 15, 0));
			// RFSHCTL3.dis_auto_refresh =0
			rddata = mmio_rd32(cfg_base + 0x60);
			rddata = modified_bits_by_value(rddata, 0, 0, 0); // RFSHCTL3.dis_auto_refresh
			mmio_wr32(cfg_base + 0x60, rddata);
		}
		uartlog("cvx16_rdlvl_req\n");
		ddr_debug_wr32(0x30);
		ddr_debug_num_write();
	}
#endif

	cvx16_rdvld_train();

	//    //RFSHCTL3.dis_auto_refresh =0
	//    rddata = mmio_rd32(cfg_base + 0x60);
	//    rddata=modified_bits_by_value(rddata, 0, 0, 0); //RFSHCTL3.dis_auto_refresh
	//    mmio_wr32(cfg_base + 0x60, rddata);
	// bist clock disable
	mmio_wr32(DDR_BIST_BASE + 0x0, 0x00040000);
	cvx16_dfi_ca_park_prbs(0);
	// restore PWRCTL.powerdown_en, PWRCTL.selfref_en
	rddata = mmio_rd32(cfg_base + 0x30);
	rddata = modified_bits_by_value(rddata, selfref_sw, 5, 5); // PWRCTL.selfref_sw
	rddata = modified_bits_by_value(rddata, en_dfi_dram_clk_disable, 3, 3); // PWRCTL.en_dfi_dram_clk_disable
	// rddata=modified_bits_by_value(rddata, 0, 2, 2); //PWRCTL.deeppowerdown_en, non-mDDR/non-LPDDR2/non-LPDDR3,
							   //this register must not be set to 1
	rddata = modified_bits_by_value(rddata, powerdown_en, 1, 1); // PWRCTL.powerdown_en
	rddata = modified_bits_by_value(rddata, selfref_en, 0, 0); // PWRCTL.selfref_en
	mmio_wr32(cfg_base + 0x30, rddata);
	// Write 1 to PCTRL_n.port_en
	for (int i = 1; i < port_num; i++) {
		mmio_wr32(cfg_base + 0x490 + 0xb0 * i, 0x1);
	}
	// cvx16_rdlvl_status();
	cvx16_clk_gating_enable();
}

void cvx16_wdqlvl_req(uint32_t data_mode, uint32_t lvl_mode)
{
	uint32_t selfref_sw;
	uint32_t en_dfi_dram_clk_disable;
	uint32_t powerdown_en;
	uint32_t selfref_en;
	// uint32_t bist_data_mode; //unused
	uint32_t port_num;
	// Note: training need ctrl_low_patch first
	//  Write 0 to PCTRL_n.port_en, without port 0
	//  port number = 0,1,2,3
	port_num = 0x4;
	for (int i = 1; i < port_num; i++) {
		mmio_wr32(cfg_base + 0x490 + 0xb0 * i, 0x0);
	}
	// Poll PSTAT.rd_port_busy_n = 0
	// Poll PSTAT.wr_port_busy_n = 0
	while (1) {
		rddata = mmio_rd32(cfg_base + 0x3fc);
		KC_MSG("Poll PSTAT.rd_port_busy_n =0\n");

		if (rddata == 0) {
			break;
		}
	}
	// disable PWRCTL.powerdown_en, PWRCTL.selfref_en
	rddata = mmio_rd32(cfg_base + 0x30);
	selfref_sw = get_bits_from_value(rddata, 5, 5);
	en_dfi_dram_clk_disable = get_bits_from_value(rddata, 3, 3);
	powerdown_en = get_bits_from_value(rddata, 1, 1);
	selfref_en = get_bits_from_value(rddata, 0, 0);
	rddata = modified_bits_by_value(rddata, 0, 5, 5); // PWRCTL.selfref_sw
	rddata = modified_bits_by_value(rddata, 0, 3, 3); // PWRCTL.en_dfi_dram_clk_disable
	// rddata=modified_bits_by_value(rddata, 0, 2, 2); //PWRCTL.deeppowerdown_en, non-mDDR/non-LPDDR2/non-LPDDR3,
							   //this register must not be set to 1
	rddata = modified_bits_by_value(rddata, 0, 1, 1); // PWRCTL.powerdown_en
	rddata = modified_bits_by_value(rddata, 0, 0, 0); // PWRCTL.selfref_en
	mmio_wr32(cfg_base + 0x30, rddata);
	cvx16_clk_gating_disable();
	uartlog("cvx16_wdqlvl_req\n");
	ddr_debug_wr32(0x31);
	ddr_debug_num_write();
	cvx16_dfi_ca_park_prbs(1);
	KC_MSG("cvx16_wdqlvl_req\n");

	// param_phyd_piwdqlvl_dq_mode
	//     <= #RD (~pwstrb_mask[12] & param_phyd_piwdqlvl_dq_mode) |  pwstrb_mask_pwdata[12];
	// param_phyd_piwdqlvl_dm_mode
	//     <= #RD (~pwstrb_mask[13] & param_phyd_piwdqlvl_dm_mode) |  pwstrb_mask_pwdata[13];
	rddata = mmio_rd32(0x00BC + PHYD_BASE_ADDR);
	// lvl_mode =0x0, wdmlvl
	// lvl_mode =0x1, wdqlvl
	// lvl_mode =0x2, wdqlvl and wdmlvl
	if (lvl_mode == 0x0) {
		rddata = modified_bits_by_value(rddata, 0, 12, 12); // param_phyd_piwdqlvl_dq_mode
		rddata = modified_bits_by_value(rddata, 1, 13, 13); // param_phyd_piwdqlvl_dm_mode
	} else if (lvl_mode == 0x1) {
		rddata = modified_bits_by_value(rddata, 1, 12, 12); // param_phyd_piwdqlvl_dq_mode
		rddata = modified_bits_by_value(rddata, 0, 13, 13); // param_phyd_piwdqlvl_dm_mode
	} else if (lvl_mode == 0x2) {
		rddata = modified_bits_by_value(rddata, 1, 12, 12); // param_phyd_piwdqlvl_dq_mode
		rddata = modified_bits_by_value(rddata, 1, 13, 13); // param_phyd_piwdqlvl_dm_mode
	}
	mmio_wr32(0x00BC + PHYD_BASE_ADDR, rddata);
	if (lvl_mode == 0x0) {
		rddata = mmio_rd32(cfg_base + 0xC);
		rddata = modified_bits_by_value(rddata, 1, 7, 7);
		mmio_wr32(cfg_base + 0xC, rddata);
		//        cvx16_bist_wdmlvl_init(sram_sp);
		cvx16_bist_wdmlvl_init();
	} else {
		// bist setting for dfi rdglvl
		// data_mode = 0x0 : phyd pattern
		// data_mode = 0x1 : bist read/write
		// data_mode = 0x11: with Error enject,  multi- bist write/read
		// data_mode = 0x12: with Error enject,  multi- bist write/read
		//         cvx16_bist_wdqlvl_init(data_mode, sram_sp);
		cvx16_bist_wdqlvl_init(data_mode);
	}
	uartlog("cvx16_wdqlvl_req\n");
	ddr_debug_wr32(0x31);
	ddr_debug_num_write();
	rddata = mmio_rd32(0x018C + PHYD_BASE_ADDR);
	rddata = modified_bits_by_value(rddata, 1, 0, 0); // param_phyd_dfi_wdqlvl_req
	if (lvl_mode == 0x0) {
		rddata = modified_bits_by_value(rddata, 0, 10, 10); // param_phyd_dfi_wdqlvl_vref_train_en
	} else {
		rddata = modified_bits_by_value(rddata, 1, 10, 10); // param_phyd_dfi_wdqlvl_vref_train_en
	}
	if ((data_mode == 0x1) || (data_mode == 0x11) || (data_mode == 0x12)) {
		rddata = modified_bits_by_value(rddata, 1, 4, 4); // param_phyd_dfi_wdqlvl_bist_data_en
	} else {
		rddata = modified_bits_by_value(rddata, 0, 4, 4); // param_phyd_dfi_wdqlvl_bist_data_en
	}
	mmio_wr32(0x018C + PHYD_BASE_ADDR, rddata);
	KC_MSG("wait retraining finish ...\n");

	while (1) {
		//[0] param_phyd_dfi_wrlvl_done
		//[1] param_phyd_dfi_rdglvl_done
		//[2] param_phyd_dfi_rdlvl_done
		//[3] param_phyd_dfi_wdqlvl_done
		rddata = mmio_rd32(0x3444 + PHYD_BASE_ADDR);
		if (get_bits_from_value(rddata, 3, 3) == 0x1) {
			break;
		}
	}
	rddata = mmio_rd32(cfg_base + 0xC);
	rddata = modified_bits_by_value(rddata, 0, 7, 7);
	mmio_wr32(cfg_base + 0xC, rddata);
	// bist clock disable
	mmio_wr32(DDR_BIST_BASE + 0x0, 0x00040000);
	cvx16_dfi_ca_park_prbs(0);
	// restore PWRCTL.powerdown_en, PWRCTL.selfref_en
	rddata = mmio_rd32(cfg_base + 0x30);
	rddata = modified_bits_by_value(rddata, selfref_sw, 5, 5); // PWRCTL.selfref_sw
	rddata = modified_bits_by_value(rddata, en_dfi_dram_clk_disable, 3, 3); // PWRCTL.en_dfi_dram_clk_disable
	// rddata=modified_bits_by_value(rddata, 0, 2, 2); //PWRCTL.deeppowerdown_en, non-mDDR/non-LPDDR2/non-LPDDR3,
							   //this register must not be set to 1
	rddata = modified_bits_by_value(rddata, powerdown_en, 1, 1); // PWRCTL.powerdown_en
	rddata = modified_bits_by_value(rddata, selfref_en, 0, 0); // PWRCTL.selfref_en
	mmio_wr32(cfg_base + 0x30, rddata);
	// Write 1 to PCTRL_n.port_en
	for (int i = 1; i < port_num; i++) {
		mmio_wr32(cfg_base + 0x490 + 0xb0 * i, 0x1);
	}
	// cvx16_wdqlvl_status();
	cvx16_clk_gating_enable();
}

void cvx16_wdqlvl_sw_req(uint32_t data_mode, uint32_t lvl_mode)
{
	uint32_t selfref_sw;
	uint32_t en_dfi_dram_clk_disable;
	uint32_t powerdown_en;
	uint32_t selfref_en;
	// uint32_t bist_data_mode; //unused
	uint32_t port_num;
	uint32_t byte0_piwdqlvl_sw_upd_ack;
	uint32_t byte1_piwdqlvl_sw_upd_ack;
	uint32_t tx_vref_sel;
	uint32_t byte0_data_rise_fail;
	uint32_t byte0_data_fall_fail;
	uint32_t byte1_data_rise_fail;
	uint32_t byte1_data_fall_fail;
	uint32_t dlie_code;
	uint32_t byte0_all_le_found;
	uint32_t byte0_all_te_found;
	uint32_t byte1_all_le_found;
	uint32_t byte1_all_te_found;
	// uint32_t sram_sp;
	//  Write 0 to PCTRL_n.port_en, without port 0
	//  port number = 0,1,2,3
	port_num = 0x4;
	for (int i = 1; i < port_num; i++) {
		mmio_wr32(cfg_base + 0x490 + 0xb0 * i, 0x0);
	}
	// Poll PSTAT.rd_port_busy_n = 0
	// Poll PSTAT.wr_port_busy_n = 0
	while (1) {
		rddata = mmio_rd32(cfg_base + 0x3fc);
		KC_MSG("Poll PSTAT.rd_port_busy_n =0\n");

		if (rddata == 0) {
			break;
		}
	}
	// disable PWRCTL.powerdown_en, PWRCTL.selfref_en
	rddata = mmio_rd32(cfg_base + 0x30);
	selfref_sw = get_bits_from_value(rddata, 5, 5);
	en_dfi_dram_clk_disable = get_bits_from_value(rddata, 3, 3);
	powerdown_en = get_bits_from_value(rddata, 1, 1);
	selfref_en = get_bits_from_value(rddata, 0, 0);
	rddata = modified_bits_by_value(rddata, 0, 5, 5); // PWRCTL.selfref_sw
	rddata = modified_bits_by_value(rddata, 0, 3, 3); // PWRCTL.en_dfi_dram_clk_disable
	// rddata=modified_bits_by_value(rddata, 0, 2, 2); //PWRCTL.deeppowerdown_en, non-mDDR/non-LPDDR2/non-LPDDR3,
							   //this register must not be set to 1
	rddata = modified_bits_by_value(rddata, 0, 1, 1); // PWRCTL.powerdown_en
	rddata = modified_bits_by_value(rddata, 0, 0, 0); // PWRCTL.selfref_en
	mmio_wr32(cfg_base + 0x30, rddata);
	cvx16_clk_gating_disable();
	uartlog("cvx16_wdqlvl_sw_req\n");
	ddr_debug_wr32(0x31);
	ddr_debug_num_write();
	cvx16_dfi_ca_park_prbs(1);
	KC_MSG("cvx16_wdqlvl_sw_req\n");

	// param_phyd_piwdqlvl_dq_mode
	//     <= #RD (~pwstrb_mask[12] & param_phyd_piwdqlvl_dq_mode) |  pwstrb_mask_pwdata[12];
	// param_phyd_piwdqlvl_dm_mode
	//     <= #RD (~pwstrb_mask[13] & param_phyd_piwdqlvl_dm_mode) |  pwstrb_mask_pwdata[13];
	rddata = mmio_rd32(0x00BC + PHYD_BASE_ADDR);
	// lvl_mode =0x0, wdmlvl
	// lvl_mode =0x1, wdqlvl
	// lvl_mode =0x2, wdqlvl and wdmlvl
	if (lvl_mode == 0x0) {
		rddata = modified_bits_by_value(rddata, 0, 12, 12); // param_phyd_piwdqlvl_dq_mode
		rddata = modified_bits_by_value(rddata, 1, 13, 13); // param_phyd_piwdqlvl_dm_mode
	} else if (lvl_mode == 0x1) {
		rddata = modified_bits_by_value(rddata, 1, 12, 12); // param_phyd_piwdqlvl_dq_mode
		rddata = modified_bits_by_value(rddata, 0, 13, 13); // param_phyd_piwdqlvl_dm_mode
	} else if (lvl_mode == 0x2) {
		rddata = modified_bits_by_value(rddata, 1, 12, 12); // param_phyd_piwdqlvl_dq_mode
		rddata = modified_bits_by_value(rddata, 1, 13, 13); // param_phyd_piwdqlvl_dm_mode
	}
	mmio_wr32(0x00BC + PHYD_BASE_ADDR, rddata);
	if (lvl_mode == 0x0) {
		//        cvx16_bist_wdmlvl_init(sram_sp);
		cvx16_bist_wdmlvl_init();
	} else {
		// bist setting for dfi rdglvl
		// data_mode = 0x0 : phyd pattern
		// data_mode = 0x1 : bist read/write
		// data_mode = 0x11: with Error enject,  multi- bist write/read
		// data_mode = 0x12: with Error enject,  multi- bist write/read
		//         cvx16_bist_wdqlvl_init(data_mode, sram_sp);
		cvx16_bist_wdqlvl_init(data_mode);
	}
	uartlog("cvx16_wdqlvl_req sw\n");
	ddr_debug_wr32(0x31);
	ddr_debug_num_write();
	// SW mode
	rddata = mmio_rd32(0x00BC + PHYD_BASE_ADDR);
	rddata = modified_bits_by_value(rddata, 1, 7, 7); // param_phyd_piwdqlvl_sw
	mmio_wr32(0x00BC + PHYD_BASE_ADDR, rddata);
	rddata = mmio_rd32(0x018C + PHYD_BASE_ADDR);
	rddata = modified_bits_by_value(rddata, 1, 0, 0); // param_phyd_dfi_wdqlvl_req
	if (lvl_mode == 0x0) {
		rddata = modified_bits_by_value(rddata, 0, 10, 10); // param_phyd_dfi_wdqlvl_vref_train_en
	} else {
		rddata = modified_bits_by_value(rddata, 1, 10, 10); // param_phyd_dfi_wdqlvl_vref_train_en
	}
	if ((data_mode == 0x1) || (data_mode == 0x11) || (data_mode == 0x12)) {
		rddata = modified_bits_by_value(rddata, 1, 4, 4); // param_phyd_dfi_wdqlvl_bist_data_en
	} else {
		rddata = modified_bits_by_value(rddata, 0, 4, 4); // param_phyd_dfi_wdqlvl_bist_data_en
	}
	mmio_wr32(0x018C + PHYD_BASE_ADDR, rddata);
	KC_MSG("wait retraining finish ...\n");

	while (1) {
		//[0] param_phyd_dfi_wrlvl_done
		//[1] param_phyd_dfi_rdglvl_done
		//[2] param_phyd_dfi_rdlvl_done
		//[3] param_phyd_dfi_wdqlvl_done
		rddata = mmio_rd32(0x3444 + PHYD_BASE_ADDR);
		if (get_bits_from_value(rddata, 3, 3) == 0x1) {
			break;
		}
		while (1) {
			rddata = mmio_rd32(0x32A4 + PHYD_BASE_ADDR);
			byte0_piwdqlvl_sw_upd_ack = get_bits_from_value(rddata, 0, 0);
			byte0_all_le_found = get_bits_from_value(rddata, 18, 18);
			byte0_all_te_found = get_bits_from_value(rddata, 17, 17);
			rddata = mmio_rd32(0x32E4 + PHYD_BASE_ADDR);
			byte1_piwdqlvl_sw_upd_ack = get_bits_from_value(rddata, 0, 0);
			byte1_all_le_found = get_bits_from_value(rddata, 18, 18);
			byte1_all_te_found = get_bits_from_value(rddata, 17, 17);
			KC_MSG("=1 byte0_piwdqlvl_sw_upd_ack = %x, byte1_piwdqlvl_sw_upd_ack = %x ",
			       byte0_piwdqlvl_sw_upd_ack, byte1_piwdqlvl_sw_upd_ack);
			KC_MSG("byte0_all_found = %x, byte1_all_found = %x\n",
			       (byte0_all_le_found & byte0_all_te_found),
			       (byte1_all_le_found & byte1_all_te_found));

			if (((byte0_piwdqlvl_sw_upd_ack == 0x1) || (byte0_all_le_found & byte0_all_te_found)) &&
			    ((byte1_piwdqlvl_sw_upd_ack == 0x1) || (byte1_all_le_found & byte1_all_te_found))) {
				rddata = mmio_rd32(0x0410 + PHYD_BASE_ADDR);
				tx_vref_sel = get_bits_from_value(rddata, 20, 16);
				rddata = mmio_rd32(0x32A8 + PHYD_BASE_ADDR);
				dlie_code = get_bits_from_value(rddata, 30, 18);
				if (byte0_all_te_found) {
					byte0_data_rise_fail = 0xff;
					byte0_data_fall_fail = 0xff;
				} else {
					byte0_data_rise_fail = get_bits_from_value(rddata, 17, 9);
					byte0_data_fall_fail = get_bits_from_value(rddata, 8, 0);
				}
				rddata = mmio_rd32(0x32E8 + PHYD_BASE_ADDR);
				if (byte1_all_te_found) {
					byte1_data_rise_fail = 0xff;
					byte1_data_fall_fail = 0xff;
				} else {
					byte1_data_rise_fail = get_bits_from_value(rddata, 17, 9);
					byte1_data_fall_fail = get_bits_from_value(rddata, 8, 0);
				}
				if ((byte0_piwdqlvl_sw_upd_ack == 0x1) || (byte1_piwdqlvl_sw_upd_ack == 0x1)) {
					SHMOO_MSG("vref = %02x, sw_wdq_training_start = %08x , ",
					       tx_vref_sel, dlie_code);

					SHMOO_MSG("err_data_rise/err_data_fall = %08x, %08x\n",
					       ((byte0_data_rise_fail & 0x000000FF) |
						((byte1_data_rise_fail & 0x000000FF) << 8)),
					       ((byte0_data_fall_fail & 0x000000FF) |
						((byte1_data_fall_fail & 0x000000FF) << 8)));
				}
				rddata = mmio_rd32(0x00BC + PHYD_BASE_ADDR);
				rddata = modified_bits_by_value(rddata, 1, 8, 8); // param_phyd_piwdqlvl_sw_upd_req
				mmio_wr32(0x00BC + PHYD_BASE_ADDR, rddata);
				KC_MSG("param_phyd_piwdqlvl_sw_upd_req = %x\n",
				       get_bits_from_value(rddata, 8, 8));

				KC_MSG("byte0_all_le_found, byte0_all_te_found, ");
				KC_MSG("byte1_all_le_found, byte1_all_te_found");
				KC_MSG("= %x %x %x %x\n",
				       byte0_all_le_found, byte0_all_te_found,
				       byte1_all_le_found, byte1_all_te_found);

				break;
			}
			if ((byte0_all_le_found == 0x1) && (byte0_all_te_found == 0x1) && (byte1_all_le_found == 0x1) &&
			    (byte1_all_te_found == 0x1)) {
				break;
			}
		}
	}
	// bist clock disable
	mmio_wr32(DDR_BIST_BASE + 0x0, 0x00040000);
	cvx16_dfi_ca_park_prbs(0);
	// restore PWRCTL.powerdown_en, PWRCTL.selfref_en
	rddata = mmio_rd32(cfg_base + 0x30);
	rddata = modified_bits_by_value(rddata, selfref_sw, 5, 5); // PWRCTL.selfref_sw
	rddata = modified_bits_by_value(rddata, en_dfi_dram_clk_disable, 3, 3); // PWRCTL.en_dfi_dram_clk_disable
	// rddata=modified_bits_by_value(rddata, 0, 2, 2); //PWRCTL.deeppowerdown_en, non-mDDR/non-LPDDR2/non-LPDDR3,
							   //this register must not be set to 1
	rddata = modified_bits_by_value(rddata, powerdown_en, 1, 1); // PWRCTL.powerdown_en
	rddata = modified_bits_by_value(rddata, selfref_en, 0, 0); // PWRCTL.selfref_en
	mmio_wr32(cfg_base + 0x30, rddata);
	// Write 1 to PCTRL_n.port_en
	for (int i = 1; i < port_num; i++) {
		mmio_wr32(cfg_base + 0x490 + 0xb0 * i, 0x1);
	}
	// cvx16_wdqlvl_status();
	cvx16_clk_gating_enable();
}


void cvx16_zqcal_status(void)
{
	uint32_t zq_drvn;
	uint32_t zq_drvp;

	uartlog("cvx16_zqcal_status\n");
	ddr_debug_wr32(0x36);
	ddr_debug_num_write();
	rddata = mmio_rd32(0x0148 + PHYD_BASE_ADDR);
	zq_drvn = get_bits_from_value(rddata, 20, 16);
	zq_drvp = get_bits_from_value(rddata, 28, 24);
	if ((zq_drvn >= 0x07) && (zq_drvn <= 0x9)) {
		KC_MSG("ZQ Complete ... param_phya_reg_tx_zq_drvn = %x\n", zq_drvn);
	} else {
		KC_MSG("ZQ Complete ... INFO! param_phya_reg_tx_zq_drvn != 0b01000 +- 1, read:value: %x\n",
		       zq_drvn);
	}
	if ((zq_drvp >= 0x07) && (zq_drvp <= 0x9)) {
		KC_MSG("ZQ Complete ... param_phya_reg_tx_zq_drvp = %x\n", zq_drvp);
	} else {
		KC_MSG("ZQ Complete ... INFO! param_phya_reg_tx_zq_drvp != 0b01000 +- 1, read value: %x\n",
		       zq_drvp);
	}
}


void cvx16_setting_check(void)
{
	uint32_t dfi_tphy_wrlat;
	uint32_t dfi_tphy_wrdata;
	uint32_t dfi_t_rddata_en;
	uint32_t dfi_t_ctrl_delay;
	uint32_t dfi_t_wrdata_delay;
	uint32_t phy_reg_version;

	uartlog("cvx16_setting_check\n");
	ddr_debug_wr32(0x0a);
	ddr_debug_num_write();
	phy_reg_version = mmio_rd32(0x3000 + PHYD_BASE_ADDR);
	rddata = mmio_rd32(cfg_base + 0x190);
	dfi_tphy_wrlat = get_bits_from_value(rddata, 5, 0); // DFITMG0.dfi_tphy_wrlat
	dfi_tphy_wrdata = get_bits_from_value(rddata, 13, 8); // DFITMG0.dfi_tphy_wrdata
	dfi_t_rddata_en = get_bits_from_value(rddata, 22, 16); // DFITMG0.dfi_t_rddata_en
	dfi_t_ctrl_delay = get_bits_from_value(rddata, 29, 24); // DFITMG0.dfi_t_ctrl_delay
	rddata = mmio_rd32(cfg_base + 0x194);
	dfi_t_wrdata_delay = get_bits_from_value(rddata, 20, 16); // DFITMG1.dfi_t_wrdata_delay
	KC_MSG("phy_reg_version = %x, dfi_t_ctrl_delay = %x, dfi_t_rddata_en = %x\n", phy_reg_version,
	       dfi_t_ctrl_delay, dfi_t_rddata_en);

	KC_MSG("dfi_tphy_wrlat  = %x, dfi_tphy_wrdata  = %x, dfi_t_wrdata_delay = %x\n", dfi_tphy_wrlat,
	       dfi_tphy_wrdata, dfi_t_wrdata_delay);

	if (dfi_t_ctrl_delay != 0x4) {
		KC_MSG("ERR !!! dfi_t_ctrl_delay not 0x4\n");
	}
#ifndef DDR2_3
#ifdef _mem_freq_2133
	if (dfi_tphy_wrlat != 0x6) {
		KC_MSG("ERR !!! dfi_tphy_wrlat not 0x6\n");
	}
	if (dfi_tphy_wrdata != 0x3) {
		KC_MSG("ERR !!! dfi_tphy_wrdata not 0x3\n");
	}
	if (dfi_t_rddata_en != 0xa) {
		KC_MSG("ERR !!! dfi_t_rddata_en not 0xa\n");
	}
	if (dfi_t_wrdata_delay != 0x7) {
		KC_MSG("ERR !!! dfi_t_wrdata_delay not 0x7\n");
	}
#endif
#ifdef _mem_freq_1866
	if (dfi_tphy_wrlat != 0x5) {
		KC_MSG("ERR !!! dfi_tphy_wrlat not 0x5\n");
	}
	if (dfi_tphy_wrdata != 0x3) {
		KC_MSG("ERR !!! dfi_tphy_wrdata not 0x3\n");
	}
	if (dfi_t_rddata_en != 0xa) {
		KC_MSG("ERR !!! dfi_t_rddata_en not 0xa\n");
	}
	if (dfi_t_wrdata_delay != 0x7) {
		KC_MSG("ERR !!! dfi_t_wrdata_delay not 0x7\n");
	}
#endif
#ifdef _mem_freq_1600
	if (dfi_tphy_wrlat != 0x4) {
		KC_MSG("ERR !!! dfi_tphy_wrlat not 0x4\n");
	}
	if (dfi_tphy_wrdata != 0x3) {
		KC_MSG("ERR !!! dfi_tphy_wrdata not 0x3\n");
	}
	if (dfi_t_rddata_en != 0x8) {
		KC_MSG("ERR !!! dfi_t_rddata_en not 0x8\n");
	}
	if (dfi_t_wrdata_delay != 0x7) {
		KC_MSG("ERR !!! dfi_t_wrdata_delay not 0x7\n");
	}
#endif
#ifdef _mem_freq_1333
#ifdef ESMT_ETRON_1333
	if (dfi_tphy_wrlat != 0x4) {
		KC_MSG("ERR !!! dfi_tphy_wrlat not 0x4\n");
	}
	if (dfi_tphy_wrdata != 0x3) {
		KC_MSG("ERR !!! dfi_tphy_wrdata not 0x3\n");
	}
	if (dfi_t_rddata_en != 0x7) {
		KC_MSG("ERR !!! dfi_t_rddata_en not 0x7\n");
	}
	if (dfi_t_wrdata_delay != 0x7) {
		KC_MSG("ERR !!! dfi_t_wrdata_delay not 0x7\n");
	}
#else
	if (dfi_tphy_wrlat != 0x2) {
		KC_MSG("ERR !!! dfi_tphy_wrlat not 0x2\n");
	}
	if (dfi_tphy_wrdata != 0x3) {
		KC_MSG("ERR !!! dfi_tphy_wrdata not 0x3\n");
	}
	if (dfi_t_rddata_en != 0x5) {
		KC_MSG("ERR !!! dfi_t_rddata_en not 0x5\n");
	}
	if (dfi_t_wrdata_delay != 0x7) {
		KC_MSG("ERR !!! dfi_t_wrdata_delay not 0x7\n");
	}
#endif
#endif
#ifdef _mem_freq_1066
	if (dfi_tphy_wrlat != 0x2) {
		KC_MSG("ERR !!! dfi_tphy_wrlat not 0x2\n");
	}
	if (dfi_tphy_wrdata != 0x3) {
		KC_MSG("ERR !!! dfi_tphy_wrdata not 0x3\n");
	}
	if (dfi_t_rddata_en != 0x5) {
		KC_MSG("ERR !!! dfi_t_rddata_en not 0x5\n");
	}
	if (dfi_t_wrdata_delay != 0x7) {
		KC_MSG("ERR !!! dfi_t_wrdata_delay not 0x7\n");
	}
#endif
#else
	if (get_ddr_type() == DDR_TYPE_DDR3) {//DDR3:1866

		if (dfi_tphy_wrlat != 0x5) {
			KC_MSG("ERR !!! dfi_tphy_wrlat not 0x5\n");
		}
		if (dfi_tphy_wrdata != 0x3) {
			KC_MSG("ERR !!! dfi_tphy_wrdata not 0x3\n");
		}
		if (dfi_t_rddata_en != 0xa) {
			KC_MSG("ERR !!! dfi_t_rddata_en not 0xa\n");
		}
		if (dfi_t_wrdata_delay != 0x7) {
			KC_MSG("ERR !!! dfi_t_wrdata_delay not 0x7\n");
		}
	} else {
	#ifdef ESMT_ETRON_1333
		if (dfi_tphy_wrlat != 0x4) {
			KC_MSG("ERR !!! dfi_tphy_wrlat not 0x4\n");
		}
		if (dfi_tphy_wrdata != 0x3) {
			KC_MSG("ERR !!! dfi_tphy_wrdata not 0x3\n");
		}
		if (dfi_t_rddata_en != 0x7) {
			KC_MSG("ERR !!! dfi_t_rddata_en not 0x7\n");
		}
		if (dfi_t_wrdata_delay != 0x7) {
			KC_MSG("ERR !!! dfi_t_wrdata_delay not 0x7\n");
		}
	#else
		if (dfi_tphy_wrlat != 0x2) {
			KC_MSG("ERR !!! dfi_tphy_wrlat not 0x2\n");
		}
		if (dfi_tphy_wrdata != 0x3) {
			KC_MSG("ERR !!! dfi_tphy_wrdata not 0x3\n");
		}
		if (dfi_t_rddata_en != 0x5) {
			KC_MSG("ERR !!! dfi_t_rddata_en not 0x5\n");
		}
		if (dfi_t_wrdata_delay != 0x7) {
			KC_MSG("ERR !!! dfi_t_wrdata_delay not 0x7\n");
		}
	#endif
	}
#endif
}


void cvx16_set_dq_vref(uint32_t vref)
{
#ifdef DDR4
	uint32_t en_dfi_dram_clk_disable;
	uint32_t powerdown_en;
	uint32_t selfref_en;
	uint32_t mr6_tmp;
#endif //DDR4
	uartlog("cvx16_set_dq_vref\n");
	ddr_debug_wr32(0x3b);
	ddr_debug_num_write();
#ifdef DDR4
	// save lowpower setting
	rddata = mmio_rd32(cfg_base + 0x30);
	// en_dfi_dram_clk_disable = rddata[3];
	//  powerdown_en            = rddata[1];
	//  selfref_en              = rddata[0];
	en_dfi_dram_clk_disable = get_bits_from_value(rddata, 3, 3);
	powerdown_en = get_bits_from_value(rddata, 1, 1);
	selfref_en = get_bits_from_value(rddata, 0, 0);
	rddata = modified_bits_by_value(rddata, 0, 3, 3);
	rddata = modified_bits_by_value(rddata, 0, 1, 1);
	rddata = modified_bits_by_value(rddata, 0, 0, 0);
	mmio_wr32(cfg_base + 0x30, rddata);
	// dis_auto_refresh = 1
	rddata = mmio_rd32(cfg_base + 0x60);
	rddata = modified_bits_by_value(rddata, 1, 0, 0);
	mmio_wr32(cfg_base + 0x60, rddata);
	// read mr6 @INIT7
	rddata = mmio_rd32(cfg_base + 0xec);
	rddata = modified_bits_by_value(rddata, vref, 6, 0);
	mr6_tmp = modified_bits_by_value(rddata, 1, 7, 7);
	cvx16_synp_mrw(0x6, mr6_tmp);
	uartlog("vrefDQ Training Enable\n");
	KC_MSG("vrefDQ Training Enable\n");

	opdelay(150);
	cvx16_synp_mrw(0x6, mr6_tmp);
	uartlog("vrefDQ set\n");
	opdelay(150);
	mr6_tmp = modified_bits_by_value(mr6_tmp, 0, 7, 7);
	cvx16_synp_mrw(0x6, mr6_tmp);
	uartlog("vrefDQ Training disable\n");
	KC_MSG("vrefDQ Training disable\n");

	opdelay(150);
	// dis_auto_refresh = 0
	rddata = mmio_rd32(cfg_base + 0x60);
	rddata = modified_bits_by_value(rddata, 0, 0, 0);
	mmio_wr32(cfg_base + 0x60, rddata);
	// restore
	rddata = mmio_rd32(cfg_base + 0x30);
	// rddata[5] = 0x0; //PWRCTL.selfref_sw
	rddata = modified_bits_by_value(rddata, 0, 5, 5);
	// rddata[3] = en_dfi_dram_clk_disable; //PWRCTL.en_dfi_dram_clk_disable
	rddata = modified_bits_by_value(rddata, en_dfi_dram_clk_disable, 3, 3);
	// rddata[1] = powerdown_en; //PWRCTL.powerdown_en
	rddata = modified_bits_by_value(rddata, powerdown_en, 1, 1);
	// rddata[0] = selfref_en; //PWRCTL.selfref_en
	rddata = modified_bits_by_value(rddata, selfref_en, 0, 0);
	mmio_wr32(cfg_base + 0x30, rddata);
	uartlog("restore selfref_en powerdown_en\n");
#endif
#ifdef DDR3
	// f0_param_phya_reg_tx_vref_sel	[20:16]
	rddata = mmio_rd32(0x0410 + PHYD_BASE_ADDR);
	rddata = modified_bits_by_value(rddata, vref, 20, 16);
	mmio_wr32(0x0410 + PHYD_BASE_ADDR, rddata);
#endif
#ifdef DDR2
	// f0_param_phya_reg_tx_vref_sel	[20:16]
	rddata = mmio_rd32(0x0410 + PHYD_BASE_ADDR);
	rddata = modified_bits_by_value(rddata, vref, 20, 16);
	mmio_wr32(0x0410 + PHYD_BASE_ADDR, rddata);
#endif
#ifdef DDR2_3
	if (get_ddr_type() == DDR_TYPE_DDR3) {
		// f0_param_phya_reg_tx_vref_sel	[20:16]
		rddata = mmio_rd32(0x0410 + PHYD_BASE_ADDR);
		rddata = modified_bits_by_value(rddata, vref, 20, 16);
		mmio_wr32(0x0410 + PHYD_BASE_ADDR, rddata);
	}
	if (get_ddr_type() == DDR_TYPE_DDR2)	{
		// f0_param_phya_reg_tx_vref_sel	[20:16]
		rddata = mmio_rd32(0x0410 + PHYD_BASE_ADDR);
		rddata = modified_bits_by_value(rddata, vref, 20, 16);
		mmio_wr32(0x0410 + PHYD_BASE_ADDR, rddata);
	}
#endif
}

void cvx16_set_dfi_init_start(void)
{
	// synp setting
	// phy is ready for initial dfi_init_start request
	// set umctl2 to tigger dfi_init_start
	uartlog("cvx16_set_dfi_init_start\n");
	ddr_debug_wr32(0x0d);
	ddr_debug_num_write();
	mmio_wr32(cfg_base + 0x00000320, 0x00000000);
	rddata = mmio_rd32(cfg_base + 0x000001b0); // dfi_init_start @ rddata[5];
	rddata = modified_bits_by_value(rddata, 1, 5, 5);
	mmio_wr32(cfg_base + 0x000001b0, rddata);
	mmio_wr32(cfg_base + 0x00000320, 1);
	KC_MSG("dfi_init_start finish\n");
}


void cvx16_dll_sw_upd(void)
{
	uartlog("cvx16_dll_sw_upd\n");
	ddr_debug_wr32(0x4B);
	ddr_debug_num_write();
	rddata = 0x1;
	mmio_wr32(0x0170 + PHYD_BASE_ADDR, rddata);
	while (1) {
		rddata = mmio_rd32(0x302C + PHYD_BASE_ADDR);
		if (rddata == 0xffffffff) {
			break;
		}
		KC_MSG("DLL SW UPD finish\n");
	}
}


void cvx16_pll_init(void)
{
	// opdelay(10);
	uartlog("pll_init\n");
	ddr_debug_wr32(0x00);
	ddr_debug_num_write();
	// TX_VREF_PD
	rddata = 0x00000000;
	mmio_wr32(0x28 + CV_DDR_PHYD_APB, rddata);
	// ZQ_240 OPTION
	rddata = 0x00080001;
	mmio_wr32(0x54 + CV_DDR_PHYD_APB, rddata);
#ifdef DDR3
// GPO setting
#ifdef _mem_freq_2133
	rddata = 0x01000808; // TOP_REG_TX_DDR3_GPO_IN =0
	mmio_wr32(0x58 + CV_DDR_PHYD_APB, rddata);
#else
	rddata = 0x01010808; // TOP_REG_TX_DDR3_GPO_IN =1
	mmio_wr32(0x58 + CV_DDR_PHYD_APB, rddata);
#endif
#endif
	
#ifdef DDR2_3
	if (get_ddr_type() == DDR_TYPE_DDR3) {
		rddata = 0x01010808; // TOP_REG_TX_DDR3_GPO_IN =1
		mmio_wr32(0x58 + CV_DDR_PHYD_APB, rddata);
	}
#endif
#ifdef SSC_EN
	//==============================================================
	// Enable SSC
	//==============================================================
	rddata = reg_set; // TOP_REG_SSC_SET
	mmio_wr32(0x54 + 0x03002900, rddata);
	rddata = get_bits_from_value(reg_span, 15, 0); // TOP_REG_SSC_SPAN
	mmio_wr32(0x58 + 0x03002900, rddata);
	rddata = get_bits_from_value(reg_step, 23, 0); // TOP_REG_SSC_STEP
	mmio_wr32(0x5C + 0x03002900, rddata);
	KC_MSG("reg_step = %lx\n", reg_step);

	rddata = mmio_rd32(0x50 + 0x03002900);
	rddata = modified_bits_by_value(rddata, ~get_bits_from_value(rddata, 0, 0), 0, 0); // TOP_REG_SSC_SW_UP
	rddata = modified_bits_by_value(rddata, 1, 1, 1); // TOP_REG_SSC_EN_SSC
	rddata = modified_bits_by_value(rddata, 0, 3, 2); // TOP_REG_SSC_SSC_MODE
	rddata = modified_bits_by_value(rddata, 0, 4, 4); // TOP_REG_SSC_BYPASS
	rddata = modified_bits_by_value(rddata, 1, 5, 5); // extpulse
	rddata = modified_bits_by_value(rddata, 0, 6, 6); // ssc_syn_fix_div
	mmio_wr32(0x50 + 0x03002900, rddata);
	uartlog("SSC_EN\n");
#else
#ifdef SSC_BYPASS
	rddata = (reg_set & 0xfc000000) + 0x04000000; // TOP_REG_SSC_SET
	mmio_wr32(0x54 + 0x03002900, rddata);
	rddata = get_bits_from_value(reg_span, 15, 0); // TOP_REG_SSC_SPAN
	mmio_wr32(0x58 + 0x03002900, rddata);
	rddata = get_bits_from_value(reg_step, 23, 0); // TOP_REG_SSC_STEP
	mmio_wr32(0x5C + 0x03002900, rddata);
	rddata = mmio_rd32(0x50 + 0x03002900);
	rddata = modified_bits_by_value(rddata, ~get_bits_from_value(rddata, 0, 0), 0, 0); // TOP_REG_SSC_SW_UP
	rddata = modified_bits_by_value(rddata, 0, 1, 1); // TOP_REG_SSC_EN_SSC
	rddata = modified_bits_by_value(rddata, 0, 3, 2); // TOP_REG_SSC_SSC_MODE
	rddata = modified_bits_by_value(rddata, 0, 4, 4); // TOP_REG_SSC_BYPASS
	rddata = modified_bits_by_value(rddata, 1, 5, 5); // TOP_REG_SSC_EXTPULSE
	rddata = modified_bits_by_value(rddata, 1, 6, 6); // ssc_syn_fix_div
	mmio_wr32(0x50 + 0x03002900, rddata);
	uartlog("SSC_BYPASS\n");
#else
	//==============================================================
	// SSC_EN =0
	//==============================================================
	uartlog("SSC_EN =0\n");
	rddata = reg_set; // TOP_REG_SSC_SET
	mmio_wr32(0x54 + 0x03002900, rddata);
	rddata = get_bits_from_value(reg_span, 15, 0); // TOP_REG_SSC_SPAN
	mmio_wr32(0x58 + 0x03002900, rddata);
	rddata = get_bits_from_value(reg_step, 23, 0); // TOP_REG_SSC_STEP
	mmio_wr32(0x5C + 0x03002900, rddata);
	rddata = mmio_rd32(0x50 + 0x03002900);
	rddata = modified_bits_by_value(rddata, ~get_bits_from_value(rddata, 0, 0), 0, 0); // TOP_REG_SSC_SW_UP
	rddata = modified_bits_by_value(rddata, 0, 1, 1); // TOP_REG_SSC_EN_SSC
	rddata = modified_bits_by_value(rddata, 0, 3, 2); // TOP_REG_SSC_SSC_MODE
	rddata = modified_bits_by_value(rddata, 0, 4, 4); // TOP_REG_SSC_BYPASS
	rddata = modified_bits_by_value(rddata, 1, 5, 5); // TOP_REG_SSC_EXTPULSE
	rddata = modified_bits_by_value(rddata, 0, 6, 6); // ssc_syn_fix_div
	mmio_wr32(0x50 + 0x03002900, rddata);
	uartlog("SSC_OFF\n");
#endif // SSC_BYPASS
#endif // SSC_EN
	// opdelay(1000);
	// DDRPLL setting
	rddata = mmio_rd32(0x0C + CV_DDR_PHYD_APB);
	//[0]    = 1;      //TOP_REG_DDRPLL_EN_DLLCLK
	//[1]    = 1;      //TOP_REG_DDRPLL_EN_LCKDET
	//[2]    = 0;      //TOP_REG_DDRPLL_EN_TST
	//[5:3]  = 0b001; //TOP_REG_DDRPLL_ICTRL
	//[6]    = 0;      //TOP_REG_DDRPLL_MAS_DIV_SEL
	//[7]    = 0;      //TOP_REG_DDRPLL_MAS_RSTZ_DIV
	//[8]    = 1;      //TOP_REG_DDRPLL_SEL_4BIT
	//[10:9] = 0b01;  //TOP_REG_DDRPLL_SEL_MODE
	//[12:11]= 0b00;  //Rev
	//[13]   = 0;      //TOP_REG_DDRPLL_SEL_LOW_SPEED
	//[14]   = 0;      //TOP_REG_DDRPLL_MAS_DIV_OUT_SEL
	//[15]   = 0;      //TOP_REG_DDRPLL_PD
	rddata = modified_bits_by_value(rddata, 0x030b, 15, 0);
	mmio_wr32(0x0C + CV_DDR_PHYD_APB, rddata);
	rddata = mmio_rd32(0x10 + CV_DDR_PHYD_APB);
	//[7:0] = 0x0;   //TOP_REG_DDRPLL_TEST
	rddata = modified_bits_by_value(rddata, 0, 7, 0); // TOP_REG_DDRPLL_TEST
	mmio_wr32(0x10 + CV_DDR_PHYD_APB, rddata);
	//[0]   = 1;    //TOP_REG_RESETZ_DIV
	rddata = 0x1;
	mmio_wr32(0x04 + CV_DDR_PHYD_APB, rddata);
	uartlog("RSTZ_DIV=1\n");
	rddata = mmio_rd32(0x0C + CV_DDR_PHYD_APB);
	//[7]   = 1;    //TOP_REG_DDRPLL_MAS_RSTZ_DIV
	rddata = modified_bits_by_value(rddata, 1, 7, 7);
	mmio_wr32(0x0C + CV_DDR_PHYD_APB, rddata);
	KC_MSG("Wait for DRRPLL LOCK=1... pll init\n");

	uartlog("Start DRRPLL LOCK pll init\n");
#ifdef REAL_LOCK
	while (1) {
		rddata = mmio_rd32(0x10 + CV_DDR_PHYD_APB);
		if (get_bits_from_value(rddata, 15, 15)) {
			break;
		}
	}
#else
	KC_MSG("check PLL lock...  pll init\n");

#endif
	uartlog("End DRRPLL LOCK=1... pll init\n");
	KC_MSG("PLL init finish !!!\n");
}


void cvx16_clk_gating_disable(void)
{
	uartlog("%s\n", __func__);
	ddr_debug_wr32(0x4C);
	ddr_debug_num_write();
	// TOP_REG_CG_EN_PHYD_TOP      0
	// TOP_REG_CG_EN_CALVL         1
	// TOP_REG_CG_EN_WRLVL         2
	// N/A                         3
	// TOP_REG_CG_EN_WRDQ          4
	// TOP_REG_CG_EN_RDDQ          5
	// TOP_REG_CG_EN_PIGTLVL       6
	// TOP_REG_CG_EN_RGTRACK       7
	// TOP_REG_CG_EN_DQSOSC        8
	// TOP_REG_CG_EN_LB            9
	// TOP_REG_CG_EN_DLL_SLAVE     10 //0:a-on
	// TOP_REG_CG_EN_DLL_MST       11 //0:a-on
	// TOP_REG_CG_EN_ZQ            12
	// TOP_REG_CG_EN_PHY_PARAM     13 //0:a-on
	// TOP_REG_CG_EN_PHYA_GATED    14 //1: clock gating enable from controller
	//                                  0:from TOP_REG_DDRPLL_CLK_OUT_GATED_EN
	// 0b01001011110101
	rddata = 0x000012F5;
	mmio_wr32(0x44 + CV_DDR_PHYD_APB, rddata);
	rddata = 0x00000000;
	mmio_wr32(0x00F4 + PHYD_BASE_ADDR, rddata); // PHYD_SHIFT_GATING_EN
	rddata = mmio_rd32(cfg_base + 0x30); // phyd_stop_clk
	rddata = modified_bits_by_value(rddata, 0, 9, 9); //en_phyd_stop_clk
	//rddata = modified_bits_by_value(rddata, 0, 10, 10 ); //phyd_stop_clk_sel, 1: old, 0: new
	rddata = modified_bits_by_value(rddata, 0, 11, 11); //en_phya_stop_clk
	//rddata = modified_bits_by_value(rddata, 0, 12, 12 ); //phya_stop_clk_sel, 1: old, 0: new
	mmio_wr32(cfg_base + 0x30, rddata);
	rddata = mmio_rd32(cfg_base + 0x148); // dfi read/write clock gatting
	rddata = modified_bits_by_value(rddata, 0, 23, 23);
	rddata = modified_bits_by_value(rddata, 0, 31, 31);
	mmio_wr32(cfg_base + 0x148, rddata);
	KC_MSG("clk_gating_disable\n");

	// disable clock gating
	// mmio_wr32(0x0800_a000 + 0x14 , 0x00000fff);
	// KC_MSG("axi disable clock gating\n");
}

void cvx16_clk_gating_enable(void)
{
	uartlog("%s\n", __func__);
	ddr_debug_wr32(0x4D);
	ddr_debug_num_write();
	// TOP_REG_CG_EN_PHYD_TOP      0
	// TOP_REG_CG_EN_CALVL         1
	// TOP_REG_CG_EN_WRLVL         2
	// N/A                         3
	// TOP_REG_CG_EN_WRDQ          4
	// TOP_REG_CG_EN_RDDQ          5
	// TOP_REG_CG_EN_PIGTLVL       6
	// TOP_REG_CG_EN_RGTRACK       7
	// TOP_REG_CG_EN_DQSOSC        8
	// TOP_REG_CG_EN_LB            9
	// TOP_REG_CG_EN_DLL_SLAVE     10 //0:a-on
	// TOP_REG_CG_EN_DLL_MST       11 //0:a-on
	// TOP_REG_CG_EN_ZQ            12
	// TOP_REG_CG_EN_PHY_PARAM     13 //0:a-on
	// TOP_REG_CG_EN_PHYA_GATED    14 //1: clock gating enable from controller
	//                                  0:from TOP_REG_DDRPLL_CLK_OUT_GATED_EN
	// 0b10110010000001
	rddata = 0x00006C81;
	mmio_wr32(0x44 + CV_DDR_PHYD_APB, rddata);
	//    #ifdef _mem_freq_1333
	//    #ifdef DDR2
	rddata = mmio_rd32(cfg_base + 0x190);
	rddata = modified_bits_by_value(rddata, 6, 28, 24);
	mmio_wr32(cfg_base + 0x190, rddata);
	//    #endif
	rddata = 0x00030033;
	mmio_wr32(0x00F4 + PHYD_BASE_ADDR, rddata); // PHYD_SHIFT_GATING_EN
	rddata = mmio_rd32(cfg_base + 0x30); // phyd_stop_clk
	rddata = modified_bits_by_value(rddata, 1, 9, 9); //en_phyd_stop_clk
	rddata = modified_bits_by_value(rddata, 0, 10, 10); //phyd_stop_clk_sel, 1: old, 0: new
	rddata = modified_bits_by_value(rddata, 1, 11, 11); //en_phya_stop_clk
	rddata = modified_bits_by_value(rddata, 0, 12, 12); //phya_stop_clk_sel, 1: old, 0: new
	mmio_wr32(cfg_base + 0x30, rddata);
	rddata = mmio_rd32(cfg_base + 0x148); // dfi read/write clock gatting
	rddata = modified_bits_by_value(rddata, 1, 23, 23);
	rddata = modified_bits_by_value(rddata, 1, 31, 31);
	mmio_wr32(cfg_base + 0x148, rddata);
	KC_MSG("clk_gating_enable\n");

	// disable clock gating
	// mmio_wr32(0x0800_a000 + 0x14 , 0x00000fff);
	// KC_MSG("axi disable clock gating\n");
}

void cvx16_dfi_phyupd_req(void)
{
	uint32_t ca_raw_upd;
	uint32_t byte0_wr_raw_upd;
	uint32_t byte1_wr_raw_upd;
	uint32_t byte0_wdqs_raw_upd;
	uint32_t byte1_wdqs_raw_upd;
	uint32_t byte0_rd_raw_upd;
	uint32_t byte1_rd_raw_upd;
	uint32_t byte0_rdg_raw_upd;
	uint32_t byte1_rdg_raw_upd;
	uint32_t byte0_rdqs_raw_upd;
	uint32_t byte1_rdqs_raw_upd;

	ca_raw_upd = 0x00000001 << 0;
	byte0_wr_raw_upd = 0x00000001 << 4;
	byte1_wr_raw_upd = 0x00000001 << 5;
	byte0_wdqs_raw_upd = 0x00000001 << 8;
	byte1_wdqs_raw_upd = 0x00000001 << 9;
	byte0_rd_raw_upd = 0x00000001 << 12;
	byte1_rd_raw_upd = 0x00000001 << 13;
	byte0_rdg_raw_upd = 0x00000001 << 16;
	byte1_rdg_raw_upd = 0x00000001 << 17;
	byte0_rdqs_raw_upd = 0x00000001 << 20;
	byte1_rdqs_raw_upd = 0x00000001 << 21;
	uartlog("%s\n", __func__);
	ddr_debug_wr32(0x54);
	ddr_debug_num_write();
	// if ($test$plusargs("")) {
	//}
	// RAW DLINE_UPD
	rddata = ca_raw_upd;
	mmio_wr32(0x016C + PHYD_BASE_ADDR, rddata);
	rddata = mmio_rd32(0x0174 + PHYD_BASE_ADDR);
}

void cvx16_en_rec_vol_mode(void)
{
	uartlog("cvx16_en_rec_vol_mode\n");
	ddr_debug_wr32(0x54);
	ddr_debug_num_write();
#ifdef DDR2
	rddata = 0x00001001;
	mmio_wr32(0x0500 + PHYD_BASE_ADDR, rddata);
	mmio_wr32(0x0540 + PHYD_BASE_ADDR, rddata);
	KC_MSG("cvx16_en_rec_vol_mode done\n");

#endif
}

void cvx16_dll_sw_clr(void)
{
	uint32_t phyd_stop_clk;

	uartlog("%s\n", __func__);
	ddr_debug_wr32(0x56);
	ddr_debug_num_write();
	phyd_stop_clk = mmio_rd32(cfg_base + 0x30); // phyd_stop_clk
	rddata = modified_bits_by_value(phyd_stop_clk, 0, 9, 9);
	mmio_wr32(cfg_base + 0x30, rddata);
	// param_phyd_sw_dfi_phyupd_req
	rddata = 0x00000101;
	mmio_wr32(0x0174 + PHYD_BASE_ADDR, rddata);
	while (1) {
		// param_phyd_to_reg_sw_phyupd_dline_done
		rddata = mmio_rd32(0x3030 + PHYD_BASE_ADDR);
		if (get_bits_from_value(rddata, 24, 24) == 0x1) {
			break;
		}
	}
	mmio_wr32(cfg_base + 0x30, phyd_stop_clk);
}


void ctrl_init_detect_dram_size(uint8_t *dram_cap_in_mbyte)
{
	uint8_t cap_in_mbyte = 0;
#ifdef DDR3
	uint32_t cmd[6];
	uint8_t i;

	// dram_cap_in_mbyte = 4;
	cap_in_mbyte = 4;

	for (i = 0; i < 6; i++)
		cmd[i] = 0x0;

	// Axsize = 3, axlen = 4, cgen
	mmio_wr32(DDR_BIST_BASE + 0x0, 0x000e0006);

	// DDR space
	mmio_wr32(DDR_BIST_BASE + 0x10, 0x00000000);
	mmio_wr32(DDR_BIST_BASE + 0x14, 0xffffffff);

	// specified AXI address step
	mmio_wr32(DDR_BIST_BASE + 0x18, 0x00000004);

	// write PRBS to 0x0 as background {{{
	// write 16 UI prbs
	cmd[0] = (1 << 30) + (0 << 21) + (3 << 12) + (5 << 9) + (0 << 8) + (0 << 0);

	for (i = 0; i < 6; i++) {
		mmio_wr32(DDR_BIST_BASE + 0x40 + i * 4, cmd[i]);
	}

	// bist_enable
	mmio_wr32(DDR_BIST_BASE + 0x0, 0x00010001);

	// polling BIST done

	do {
		rddata = mmio_rd32(DDR_BIST_BASE + 0x80);
	} while (get_bits_from_value(rddata, 2, 2) == 0);

	// bist disable
	mmio_wr32(DDR_BIST_BASE + 0x0, 0x00010000);
	// }}}

	do {
		// *dram_cap_in_mbyte++;
		cap_in_mbyte++;
		uartlog("cap_in_mbyte =  %x\n", cap_in_mbyte);

		// write ~PRBS to (0x1 << *dram_cap_in_mbyte) {{{

		// DDR space
		mmio_wr32(DDR_BIST_BASE + 0x10, 1 << (cap_in_mbyte + 20 - 4));
		// write 16 UI ~prbs
		cmd[0] = (1 << 30) + (0 << 21) + (3 << 12) + (5 << 9) + (1 << 8) + (0 << 0);

		for (i = 0; i < 6; i++) {
			mmio_wr32(DDR_BIST_BASE + 0x40 + i * 4, cmd[i]);
		}

		// bist_enable
		mmio_wr32(DDR_BIST_BASE + 0x0, 0x00010001);
		// polling BIST done

		do {
			rddata = mmio_rd32(DDR_BIST_BASE + 0x80);
		} while (get_bits_from_value(rddata, 2, 2) == 0);

		// bist disable
		mmio_wr32(DDR_BIST_BASE + 0x0, 0x00010000);
		// }}}

		// check PRBS at 0x0 {{{

		// DDR space
		mmio_wr32(DDR_BIST_BASE + 0x10, 0x00000000);
		cmd[0] = (2 << 30) + (0 << 21) + (3 << 12) + (5 << 9) + (0 << 8) + (0 << 0); // read 16 UI prbs

		for (i = 0; i < 6; i++) {
			mmio_wr32(DDR_BIST_BASE + 0x40 + i * 4, cmd[i]);
		}

		// bist_enable
		mmio_wr32(DDR_BIST_BASE + 0x0, 0x00010001);
		// polling BIST done

		do {
			rddata = mmio_rd32(DDR_BIST_BASE + 0x80);
		} while (get_bits_from_value(rddata, 2, 2) == 0);

		// bist disable
		mmio_wr32(DDR_BIST_BASE + 0x0, 0x00010000);
		// }}}

	} while ((get_bits_from_value(rddata, 3, 3) == 0) && (cap_in_mbyte < 15)); // BIST fail stop the loop

#endif
#ifdef DDR2
	// fix size for DDR2
	cap_in_mbyte = 6;
#endif
#ifdef DDR2_3
	if (get_ddr_type() == DDR_TYPE_DDR3) {
		uint32_t cmd[6];
		uint8_t i;

		// dram_cap_in_mbyte = 4;
		cap_in_mbyte = 4;

		for (i = 0; i < 6; i++)
			cmd[i] = 0x0;

		// Axsize = 3, axlen = 4, cgen
		mmio_wr32(DDR_BIST_BASE + 0x0, 0x000e0006);

		// DDR space
		mmio_wr32(DDR_BIST_BASE + 0x10, 0x00000000);
		mmio_wr32(DDR_BIST_BASE + 0x14, 0xffffffff);

		// specified AXI address step
		mmio_wr32(DDR_BIST_BASE + 0x18, 0x00000004);

		// write PRBS to 0x0 as background {{{

		cmd[0] = (1 << 30) + (0 << 21) + (3 << 12) + (5 << 9) + (0 << 8) + (0 << 0); // write 16 UI prbs

		for (i = 0; i < 6; i++) {
			mmio_wr32(DDR_BIST_BASE + 0x40 + i * 4, cmd[i]);
		}

		// bist_enable
		mmio_wr32(DDR_BIST_BASE + 0x0, 0x00010001);

		// polling BIST done

		do {
			rddata = mmio_rd32(DDR_BIST_BASE + 0x80);
		} while (get_bits_from_value(rddata, 2, 2) == 0);

		// bist disable
		mmio_wr32(DDR_BIST_BASE + 0x0, 0x00010000);
		// }}}

		do {
			// *dram_cap_in_mbyte++;
			cap_in_mbyte++;
			uartlog("cap_in_mbyte =  %x\n", cap_in_mbyte);

			// write ~PRBS to (0x1 << *dram_cap_in_mbyte) {{{

			// DDR space
			mmio_wr32(DDR_BIST_BASE + 0x10, 1 << (cap_in_mbyte + 20 - 4));
			// write 16 UI ~prbs
			cmd[0] = (1 << 30) + (0 << 21) + (3 << 12) + (5 << 9) + (1 << 8) + (0 << 0);

			for (i = 0; i < 6; i++) {
				mmio_wr32(DDR_BIST_BASE + 0x40 + i * 4, cmd[i]);
			}

			// bist_enable
			mmio_wr32(DDR_BIST_BASE + 0x0, 0x00010001);
			// polling BIST done

			do {
				rddata = mmio_rd32(DDR_BIST_BASE + 0x80);
			} while (get_bits_from_value(rddata, 2, 2) == 0);

			// bist disable
			mmio_wr32(DDR_BIST_BASE + 0x0, 0x00010000);
			// }}}

			// check PRBS at 0x0 {{{

			// DDR space
			mmio_wr32(DDR_BIST_BASE + 0x10, 0x00000000);
			// read 16 UI prbs
			cmd[0] = (2 << 30) + (0 << 21) + (3 << 12) + (5 << 9) + (0 << 8) + (0 << 0);

			for (i = 0; i < 6; i++) {
				mmio_wr32(DDR_BIST_BASE + 0x40 + i * 4, cmd[i]);
			}

			// bist_enable
			mmio_wr32(DDR_BIST_BASE + 0x0, 0x00010001);
			// polling BIST done

			do {
				rddata = mmio_rd32(DDR_BIST_BASE + 0x80);
			} while (get_bits_from_value(rddata, 2, 2) == 0);

			// bist disable
			mmio_wr32(DDR_BIST_BASE + 0x0, 0x00010000);
			// }}}

		} while ((get_bits_from_value(rddata, 3, 3) == 0) && (cap_in_mbyte < 15)); // BIST fail stop the loop
	}
	if (get_ddr_type() == DDR_TYPE_DDR2)	{
		// fix size for DDR2
		cap_in_mbyte = 6;
	}
#endif

	*dram_cap_in_mbyte = cap_in_mbyte;

	// save dram_cap_in_mbyte
	rddata = cap_in_mbyte;
	mmio_wr32(0x0208 + PHYD_BASE_ADDR, rddata);

	// cgen disable
	mmio_wr32(DDR_BIST_BASE + 0x0, 0x00040000);
}

#pragma GCC diagnostic pop
